etan39
/
cloudstack
mirror of https://github.com/apache/cloudstack.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

DPDK vHost User mode selection (#3153) 

* DPDK vHost User mode selection

* SQL text field and DPDK classes refactor

* Fix NullPointerException after refactor

* Fix unit test

* Refactor details type
This commit is contained in:
Nicolas Vazquez 2019-05-29 08:36:33 -03:00 committed by GitHub
parent 4e8f14975a
commit 501aa7cd91
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 615 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
engine/schema/src/main/resources/META-INF/db/schema-41200to41300.sql
View File

@ -19,3 +19,5 @@
-- Schema upgrade from 4.12.0.0 to 4.13.0.0
--;
-- DPDK client and server mode support
ALTER TABLE `cloud`.`service_offering_details` CHANGE COLUMN `value` `value` TEXT NOT NULL;

56
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/dpdk/DPDKDriver.java Normal file
View File

@ -0,0 +1,56 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package com.cloud.hypervisor.kvm.dpdk;
import com.cloud.utils.component.Adapter;
import java.util.Map;
public interface DPDKDriver extends Adapter {
/**
* Get the next DPDK port name to be created
*/
String getNextDpdkPort();
/**
* Get the latest DPDK port number created on a DPDK enabled host
*/
int getDpdkLatestPortNumberUsed();
/**
* Add OVS port (if it does not exist) to bridge with DPDK support
*/
void addDpdkPort(String bridgeName, String port, String vlan, DPDKHelper.VHostUserMode vHostUserMode, String dpdkOvsPath);
/**
* Since DPDK user client/server mode, retrieve the guest interfaces mode from the DPDK vHost User mode
*/
String getGuestInterfacesModeFromDPDKVhostUserMode(DPDKHelper.VHostUserMode dpdKvHostUserMode);
/**
* Get DPDK vHost User mode from extra config. If it is not present, server is returned as default
*/
DPDKHelper.VHostUserMode getDPDKvHostUserMode(Map<String, String> extraConfig);
/**
* Check for additional extra 'dpdk-interface' configurations, return them appended
*/
String getExtraDpdkProperties(Map<String, String> extraConfig);
}

115
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/dpdk/DPDKDriverImpl.java Normal file
View File

@ -0,0 +1,115 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package com.cloud.hypervisor.kvm.dpdk;
import com.cloud.utils.component.AdapterBase;
import com.cloud.utils.script.Script;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import java.util.Map;
public class DPDKDriverImpl extends AdapterBase implements DPDKDriver {
static final String DPDK_PORT_PREFIX = "csdpdk-";
private final String dpdkPortVhostUserType = "dpdkvhostuser";
private final String dpdkPortVhostUserClientType = "dpdkvhostuserclient";
private static final Logger s_logger = Logger.getLogger(DPDKDriver.class);
public DPDKDriverImpl() {
}
/**
* Get the next DPDK port name to be created
*/
public String getNextDpdkPort() {
int portNumber = getDpdkLatestPortNumberUsed();
return DPDK_PORT_PREFIX + String.valueOf(portNumber + 1);
}
/**
* Get the latest DPDK port number created on a DPDK enabled host
*/
public int getDpdkLatestPortNumberUsed() {
s_logger.debug("Checking the last DPDK port created");
String cmd = "ovs-vsctl show | grep Port | grep " + DPDK_PORT_PREFIX + " | " +
"awk '{ print $2 }' | sort -rV | head -1";
String port = Script.runSimpleBashScript(cmd);
int portNumber = 0;
if (StringUtils.isNotBlank(port)) {
String unquotedPort = port.replace("\"", "");
String dpdkPortNumber = unquotedPort.split(DPDK_PORT_PREFIX)[1];
portNumber = Integer.valueOf(dpdkPortNumber);
}
return portNumber;
}
/**
* Add OVS port (if it does not exist) to bridge with DPDK support
*/
public void addDpdkPort(String bridgeName, String port, String vlan, DPDKHelper.VHostUserMode vHostUserMode, String dpdkOvsPath) {
String type = vHostUserMode == DPDKHelper.VHostUserMode.SERVER ?
dpdkPortVhostUserType :
dpdkPortVhostUserClientType;
StringBuilder stringBuilder = new StringBuilder();
stringBuilder.append(String.format("ovs-vsctl add-port %s %s " +
"vlan_mode=access tag=%s " +
"-- set Interface %s type=%s", bridgeName, port, vlan, port, type));
if (vHostUserMode == DPDKHelper.VHostUserMode.CLIENT) {
stringBuilder.append(String.format(" options:vhost-server-path=%s/%s",
dpdkOvsPath, port));
}
String cmd = stringBuilder.toString();
s_logger.debug("DPDK property enabled, executing: " + cmd);
Script.runSimpleBashScript(cmd);
}
/**
* Since DPDK user client/server mode, retrieve the guest interfaces mode from the DPDK vHost User mode
*/
public String getGuestInterfacesModeFromDPDKVhostUserMode(DPDKHelper.VHostUserMode dpdKvHostUserMode) {
return dpdKvHostUserMode == DPDKHelper.VHostUserMode.CLIENT ? "server" : "client";
}
/**
* Get DPDK vHost User mode from extra config. If it is not present, server is returned as default
*/
public DPDKHelper.VHostUserMode getDPDKvHostUserMode(Map<String, String> extraConfig) {
return extraConfig.containsKey(DPDKHelper.DPDK_VHOST_USER_MODE) ?
DPDKHelper.VHostUserMode.fromValue(extraConfig.get(DPDKHelper.DPDK_VHOST_USER_MODE)) :
DPDKHelper.VHostUserMode.SERVER;
}
/**
* Check for additional extra 'dpdk-interface' configurations, return them appended
*/
public String getExtraDpdkProperties(Map<String, String> extraConfig) {
StringBuilder stringBuilder = new StringBuilder();
for (String key : extraConfig.keySet()) {
if (key.startsWith(DPDKHelper.DPDK_INTERFACE_PREFIX)) {
stringBuilder.append(extraConfig.get(key));
}
}
return stringBuilder.toString();
}
}

11
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java
View File

@ -46,8 +46,8 @@ import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import com.cloud.hypervisor.kvm.dpdk.DPDKHelper;
import com.cloud.resource.RequestWrapper;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
import org.apache.cloudstack.storage.to.TemplateObjectTO;
import org.apache.cloudstack.storage.to.VolumeObjectTO;
@ -524,9 +524,6 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
protected boolean dpdkSupport = false;
protected String dpdkOvsPath;
protected static final String DPDK_NUMA = ApiConstants.EXTRA_CONFIG + "-dpdk-numa";
protected static final String DPDK_HUGE_PAGES = ApiConstants.EXTRA_CONFIG + "-dpdk-hugepages";
protected static final String DPDK_INTERFACE_PREFIX = ApiConstants.EXTRA_CONFIG + "-dpdk-interface-";
private String getEndIpFromStartIp(final String startIp, final int numIps) {
final String[] tokens = startIp.split("[.]");
@ -2073,7 +2070,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
vm.setPlatformEmulator(vmTO.getPlatformEmulator());
Map<String, String> extraConfig = vmTO.getExtraConfig();
if (dpdkSupport && (!extraConfig.containsKey(DPDK_NUMA) || !extraConfig.containsKey(DPDK_HUGE_PAGES))) {
if (dpdkSupport && (!extraConfig.containsKey(DPDKHelper.DPDK_NUMA) || !extraConfig.containsKey(DPDKHelper.DPDK_HUGE_PAGES))) {
s_logger.info("DPDK is enabled but it needs extra configurations for CPU NUMA and Huge Pages for VM deployment");
}
@ -2110,7 +2107,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
grd.setVcpuNum(vcpus);
vm.addComp(grd);
if (!extraConfig.containsKey(DPDK_NUMA)) {
if (!extraConfig.containsKey(DPDKHelper.DPDK_NUMA)) {
final CpuModeDef cmd = new CpuModeDef();
cmd.setMode(_guestCpuMode);
cmd.setModel(_guestCpuModel);
@ -2238,7 +2235,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
if (MapUtils.isNotEmpty(extraConfig)) {
StringBuilder extraConfigBuilder = new StringBuilder();
for (String key : extraConfig.keySet()) {
if (!key.startsWith(DPDK_INTERFACE_PREFIX)) {
if (!key.startsWith(DPDKHelper.DPDK_INTERFACE_PREFIX) && !key.equals(DPDKHelper.DPDK_VHOST_USER_MODE)) {
extraConfigBuilder.append(extraConfig.get(key));
}
}

8
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java
View File

@ -1017,6 +1017,7 @@ public class LibvirtVMDef {
private String _dpdkSourcePath;
private String _dpdkSourcePort;
private String _dpdkExtraLines;
private String _interfaceMode;
public void defBridgeNet(String brName, String targetBrName, String macAddr, NicModel model) {
defBridgeNet(brName, targetBrName, macAddr, model, 0);
@ -1031,7 +1032,8 @@ public class LibvirtVMDef {
_networkRateKBps = networkRateKBps;
}
public void defDpdkNet(String dpdkSourcePath, String dpdkPort, String macAddress, NicModel model, Integer networkRateKBps, String extra) {
public void defDpdkNet(String dpdkSourcePath, String dpdkPort, String macAddress, NicModel model,
Integer networkRateKBps, String extra, String interfaceMode) {
_netType = GuestNetType.VHOSTUSER;
_dpdkSourcePath = dpdkSourcePath;
_dpdkSourcePort = dpdkPort;
@ -1039,6 +1041,7 @@ public class LibvirtVMDef {
_model = model;
_networkRateKBps = networkRateKBps;
_dpdkExtraLines = extra;
_interfaceMode = interfaceMode;
}
public void defDirectNet(String sourceName, String targetName, String macAddr, NicModel model, String sourceMode) {
@ -1184,7 +1187,8 @@ public class LibvirtVMDef {
} else if (_netType == GuestNetType.DIRECT) {
netBuilder.append("<source dev='" + _sourceName + "' mode='" + _netSourceMode + "'/>\n");
} else if (_netType == GuestNetType.VHOSTUSER) {
netBuilder.append("<source type='unix' path='"+ _dpdkSourcePath + _dpdkSourcePort + "' mode='client'/>\n");
netBuilder.append("<source type='unix' path='"+ _dpdkSourcePath + _dpdkSourcePort +
"' mode='" + _interfaceMode + "'/>\n");
}
if (_networkName != null) {
netBuilder.append("<target dev='" + _networkName + "'/>\n");

74
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/OvsVifDriver.java
View File

@ -24,6 +24,9 @@ import java.util.Map;
import javax.naming.ConfigurationException;
import com.cloud.hypervisor.kvm.dpdk.DPDKDriver;
import com.cloud.hypervisor.kvm.dpdk.DPDKDriverImpl;
import com.cloud.hypervisor.kvm.dpdk.DPDKHelper;
import com.cloud.utils.exception.CloudRuntimeException;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
@ -41,8 +44,7 @@ import com.cloud.utils.script.Script;
public class OvsVifDriver extends VifDriverBase {
private static final Logger s_logger = Logger.getLogger(OvsVifDriver.class);
private int _timeout;
protected static final String DPDK_PORT_PREFIX = "csdpdk-";
private DPDKDriver dpdkDriver;
@Override
public void configure(Map<String, Object> params) throws ConfigurationException {
@ -55,6 +57,11 @@ public class OvsVifDriver extends VifDriverBase {
networkScriptsDir = "scripts/vm/network/vnet";
}
String dpdk = (String) params.get("openvswitch.dpdk.enabled");
if (StringUtils.isNotBlank(dpdk) && Boolean.parseBoolean(dpdk)) {
dpdkDriver = new DPDKDriverImpl();
}
String value = (String)params.get("scripts.timeout");
_timeout = NumbersUtil.parseInt(value, 30 * 60) * 1000;
}
@ -80,55 +87,6 @@ public class OvsVifDriver extends VifDriverBase {
s_logger.debug("done looking for pifs, no more bridges");
}
/**
* Get the latest DPDK port number created on a DPDK enabled host
*/
protected int getDpdkLatestPortNumberUsed() {
s_logger.debug("Checking the last DPDK port created");
String cmd = "ovs-vsctl show | grep Port | grep " + DPDK_PORT_PREFIX + " | " +
"awk '{ print $2 }' | sort -rV | head -1";
String port = Script.runSimpleBashScript(cmd);
int portNumber = 0;
if (StringUtils.isNotBlank(port)) {
String unquotedPort = port.replace("\"", "");
String dpdkPortNumber = unquotedPort.split(DPDK_PORT_PREFIX)[1];
portNumber = Integer.valueOf(dpdkPortNumber);
}
return portNumber;
}
/**
* Get the next DPDK port name to be created
*/
protected String getNextDpdkPort() {
int portNumber = getDpdkLatestPortNumberUsed();
return DPDK_PORT_PREFIX + String.valueOf(portNumber + 1);
}
/**
* Add OVS port (if it does not exist) to bridge with DPDK support
*/
protected void addDpdkPort(String bridgeName, String port, String vlan) {
String cmd = String.format("ovs-vsctl add-port %s %s " +
"vlan_mode=access tag=%s " +
"-- set Interface %s type=dpdkvhostuser", bridgeName, port, vlan, port);
s_logger.debug("DPDK property enabled, executing: " + cmd);
Script.runSimpleBashScript(cmd);
}
/**
* Check for additional extra 'dpdk-interface' configurations, return them appended
*/
private String getExtraDpdkProperties(Map<String, String> extraConfig) {
StringBuilder stringBuilder = new StringBuilder();
for (String key : extraConfig.keySet()) {
if (key.startsWith(LibvirtComputingResource.DPDK_INTERFACE_PREFIX)) {
stringBuilder.append(extraConfig.get(key));
}
}
return stringBuilder.toString();
}
@Override
public InterfaceDef plug(NicTO nic, String guestOsType, String nicAdapter, Map<String, String> extraConfig) throws InternalErrorException, LibvirtException {
s_logger.debug("plugging nic=" + nic);
@ -158,12 +116,18 @@ public class OvsVifDriver extends VifDriverBase {
if (trafficLabel != null && !trafficLabel.isEmpty()) {
if (_libvirtComputingResource.dpdkSupport && !nic.isDpdkDisabled()) {
s_logger.debug("DPDK support enabled: configuring per traffic label " + trafficLabel);
if (StringUtils.isBlank(_libvirtComputingResource.dpdkOvsPath)) {
String dpdkOvsPath = _libvirtComputingResource.dpdkOvsPath;
if (StringUtils.isBlank(dpdkOvsPath)) {
throw new CloudRuntimeException("DPDK is enabled on the host but no OVS path has been provided");
}
String port = getNextDpdkPort();
addDpdkPort(_pifs.get(trafficLabel), port, vlanId);
intf.defDpdkNet(_libvirtComputingResource.dpdkOvsPath, port, nic.getMac(), getGuestNicModel(guestOsType, nicAdapter), 0, getExtraDpdkProperties(extraConfig));
String port = dpdkDriver.getNextDpdkPort();
DPDKHelper.VHostUserMode dpdKvHostUserMode = dpdkDriver.getDPDKvHostUserMode(extraConfig);
dpdkDriver.addDpdkPort(_pifs.get(trafficLabel), port, vlanId, dpdKvHostUserMode, dpdkOvsPath);
String interfaceMode = dpdkDriver.getGuestInterfacesModeFromDPDKVhostUserMode(dpdKvHostUserMode);
intf.defDpdkNet(dpdkOvsPath, port, nic.getMac(),
getGuestNicModel(guestOsType, nicAdapter), 0,
dpdkDriver.getExtraDpdkProperties(extraConfig),
interfaceMode);
} else {
s_logger.debug("creating a vlan dev and bridge for guest traffic per traffic label " + trafficLabel);
intf.defBridgeNet(_pifs.get(trafficLabel), null, nic.getMac(), getGuestNicModel(guestOsType, nicAdapter), networkRateKBps);

52
plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/OvsVifDriverTest.java → plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/dpdk/DPDKDriverTest.java
View File

@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
package com.cloud.hypervisor.kvm.resource;
package com.cloud.hypervisor.kvm.dpdk;
import com.cloud.utils.script.Script;
import org.junit.Assert;
@ -30,19 +30,25 @@ import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import java.util.HashMap;
import java.util.Map;
@PrepareForTest({ Script.class })
@RunWith(PowerMockRunner.class)
public class OvsVifDriverTest {
public class DPDKDriverTest {
private static final int dpdkPortNumber = 7;
private OvsVifDriver driver = new OvsVifDriver();
private DPDKDriver driver = new DPDKDriverImpl();
private Map<String, String> extraConfig;
@Before
public void initMocks() {
MockitoAnnotations.initMocks(this);
PowerMockito.mockStatic(Script.class);
Mockito.when(Script.runSimpleBashScript(Matchers.anyString())).thenReturn(null);
extraConfig = new HashMap<>();
}
@Test
@ -53,7 +59,7 @@ public class OvsVifDriverTest {
@Test
public void testGetDpdkLatestPortNumberUsedExistingDpdkPorts() {
Mockito.when(Script.runSimpleBashScript(Matchers.anyString())).
thenReturn(OvsVifDriver.DPDK_PORT_PREFIX + String.valueOf(dpdkPortNumber));
thenReturn(DPDKDriverImpl.DPDK_PORT_PREFIX + String.valueOf(dpdkPortNumber));
Assert.assertEquals(dpdkPortNumber, driver.getDpdkLatestPortNumberUsed());
}
@ -61,15 +67,47 @@ public class OvsVifDriverTest {
public void testGetNextDpdkPortNoDpdkPorts() {
Mockito.when(Script.runSimpleBashScript(Matchers.anyString())).
thenReturn(null);
String expectedPortName = OvsVifDriver.DPDK_PORT_PREFIX + String.valueOf(1);
String expectedPortName = DPDKDriverImpl.DPDK_PORT_PREFIX + String.valueOf(1);
Assert.assertEquals(expectedPortName, driver.getNextDpdkPort());
}
@Test
public void testGetNextDpdkPortExistingDpdkPorts() {
Mockito.when(Script.runSimpleBashScript(Matchers.anyString())).
thenReturn(OvsVifDriver.DPDK_PORT_PREFIX + String.valueOf(dpdkPortNumber));
String expectedPortName = OvsVifDriver.DPDK_PORT_PREFIX + String.valueOf(dpdkPortNumber + 1);
thenReturn(DPDKDriverImpl.DPDK_PORT_PREFIX + String.valueOf(dpdkPortNumber));
String expectedPortName = DPDKDriverImpl.DPDK_PORT_PREFIX + String.valueOf(dpdkPortNumber + 1);
Assert.assertEquals(expectedPortName, driver.getNextDpdkPort());
}
@Test
public void testGetGuestInterfacesModeFromDPDKVhostUserModeClientDPDK() {
String guestMode = driver.getGuestInterfacesModeFromDPDKVhostUserMode(DPDKHelper.VHostUserMode.CLIENT);
Assert.assertEquals("server", guestMode);
}
@Test
public void testGetGuestInterfacesModeFromDPDKVhostUserModeServerDPDK() {
String guestMode = driver.getGuestInterfacesModeFromDPDKVhostUserMode(DPDKHelper.VHostUserMode.SERVER);
Assert.assertEquals("client", guestMode);
}
@Test
public void testGetDPDKvHostUserModeServerExtraConfig() {
extraConfig.put(DPDKHelper.DPDK_VHOST_USER_MODE, DPDKHelper.VHostUserMode.SERVER.toString());
DPDKHelper.VHostUserMode dpdKvHostUserMode = driver.getDPDKvHostUserMode(extraConfig);
Assert.assertEquals(DPDKHelper.VHostUserMode.SERVER, dpdKvHostUserMode);
}
@Test
public void testGetDPDKvHostUserModeServerClientExtraConfig() {
extraConfig.put(DPDKHelper.DPDK_VHOST_USER_MODE, DPDKHelper.VHostUserMode.CLIENT.toString());
DPDKHelper.VHostUserMode dpdKvHostUserMode = driver.getDPDKvHostUserMode(extraConfig);
Assert.assertEquals(DPDKHelper.VHostUserMode.CLIENT, dpdKvHostUserMode);
}
@Test
public void testGetDPDKvHostUserModeServerEmptyExtraConfig() {
DPDKHelper.VHostUserMode dpdKvHostUserMode = driver.getDPDKvHostUserMode(extraConfig);
Assert.assertEquals(DPDKHelper.VHostUserMode.SERVER, dpdKvHostUserMode);
}
}

17
server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java
View File

@ -16,8 +16,10 @@
// under the License.
package com.cloud.configuration;
import java.io.UnsupportedEncodingException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URLDecoder;
import java.sql.Date;
import java.sql.PreparedStatement;
import java.util.ArrayList;
@ -2492,17 +2494,26 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
}
detailsVO = new ArrayList<ServiceOfferingDetailsVO>();
for (final Entry<String, String> detailEntry : details.entrySet()) {
String detailEntryValue = detailEntry.getValue();
if (detailEntry.getKey().equals(GPU.Keys.pciDevice.toString())) {
if (detailEntry.getValue() == null) {
if (detailEntryValue == null) {
throw new InvalidParameterValueException("Please specify a GPU Card.");
}
}
if (detailEntry.getKey().equals(GPU.Keys.vgpuType.toString())) {
if (detailEntry.getValue() == null) {
if (detailEntryValue == null) {
throw new InvalidParameterValueException("vGPUType value cannot be null");
}
}
detailsVO.add(new ServiceOfferingDetailsVO(offering.getId(), detailEntry.getKey(), detailEntry.getValue(), true));
if (detailEntry.getKey().startsWith(ApiConstants.EXTRA_CONFIG)) {
try {
detailEntryValue = URLDecoder.decode(detailEntry.getValue(), "UTF-8");
} catch (UnsupportedEncodingException | IllegalArgumentException e) {
s_logger.error("Cannot decode extra configuration value for key: " + detailEntry.getKey() + ", skipping it");
continue;
}
}
detailsVO.add(new ServiceOfferingDetailsVO(offering.getId(), detailEntry.getKey(), detailEntryValue, true));
}
}

6
server/src/main/java/com/cloud/hypervisor/HypervisorGuruBase.java
View File

@ -72,7 +72,7 @@ public abstract class HypervisorGuruBase extends AdapterBase implements Hypervis
@Inject
private ResourceManager _resourceMgr;
@Inject
private ServiceOfferingDetailsDao _serviceOfferingDetailsDao;
protected ServiceOfferingDetailsDao _serviceOfferingDetailsDao;
@Inject
private ServiceOfferingDao _serviceOfferingDao;
@ -172,8 +172,8 @@ public abstract class HypervisorGuruBase extends AdapterBase implements Hypervis
}
// Set GPU details
ServiceOfferingDetailsVO offeringDetail = null;
if ((offeringDetail = _serviceOfferingDetailsDao.findDetail(offering.getId(), GPU.Keys.vgpuType.toString())) != null) {
ServiceOfferingDetailsVO offeringDetail = _serviceOfferingDetailsDao.findDetail(offering.getId(), GPU.Keys.vgpuType.toString());
if (offeringDetail != null) {
ServiceOfferingDetailsVO groupName = _serviceOfferingDetailsDao.findDetail(offering.getId(), GPU.Keys.pciDevice.toString());
to.setGpuDevice(_resourceMgr.getGPUDevice(vm.getHostId(), groupName.getValue(), offeringDetail.getValue()));
}

31
server/src/main/java/com/cloud/hypervisor/KVMGuru.java
View File

@ -22,6 +22,9 @@ import com.cloud.agent.api.to.VirtualMachineTO;
import com.cloud.host.HostVO;
import com.cloud.host.dao.HostDao;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.hypervisor.kvm.dpdk.DPDKHelper;
import com.cloud.offering.ServiceOffering;
import com.cloud.service.ServiceOfferingDetailsVO;
import com.cloud.storage.DataStoreRole;
import com.cloud.storage.GuestOSHypervisorVO;
import com.cloud.storage.GuestOSVO;
@ -31,13 +34,16 @@ import com.cloud.utils.Pair;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.vm.VirtualMachine;
import com.cloud.vm.VirtualMachineProfile;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.storage.command.CopyCommand;
import org.apache.cloudstack.storage.command.StorageSubSystemCommand;
import org.apache.commons.collections.CollectionUtils;
import org.apache.log4j.Logger;
import javax.inject.Inject;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.util.List;
import java.util.Map;
public class KVMGuru extends HypervisorGuruBase implements HypervisorGuru {
@ -47,6 +53,8 @@ public class KVMGuru extends HypervisorGuruBase implements HypervisorGuru {
GuestOSHypervisorDao _guestOsHypervisorDao;
@Inject
HostDao _hostDao;
@Inject
DPDKHelper dpdkHelper;
public static final Logger s_logger = Logger.getLogger(KVMGuru.class);
@ -106,6 +114,11 @@ public class KVMGuru extends HypervisorGuruBase implements HypervisorGuru {
public VirtualMachineTO implement(VirtualMachineProfile vm) {
VirtualMachineTO to = toVirtualMachineTO(vm);
setVmQuotaPercentage(to, vm);
addServiceOfferingExtraConfiguration(to, vm);
if (dpdkHelper.isDPDKvHostUserModeSettingOnServiceOffering(vm)) {
dpdkHelper.setDpdkVhostUserMode(to, vm);
}
// Determine the VM's OS description
GuestOSVO guestOS = _guestOsDao.findByIdIncludingRemoved(vm.getVirtualMachine().getGuestOSId());
@ -124,6 +137,24 @@ public class KVMGuru extends HypervisorGuruBase implements HypervisorGuru {
return to;
}
/**
* Add extra configurations from service offering to the VM TO.
* Extra configuration keys are expected in formats:
* - "extraconfig-N"
* - "extraconfig-CONFIG_NAME"
*/
protected void addServiceOfferingExtraConfiguration(VirtualMachineTO to, VirtualMachineProfile vmProfile) {
ServiceOffering offering = vmProfile.getServiceOffering();
List<ServiceOfferingDetailsVO> details = _serviceOfferingDetailsDao.listDetails(offering.getId());
if (CollectionUtils.isNotEmpty(details)) {
for (ServiceOfferingDetailsVO detail : details) {
if (detail.getName().startsWith(ApiConstants.EXTRA_CONFIG)) {
to.addExtraConfig(detail.getName(), detail.getValue());
}
}
}
}
@Override
public Pair<Boolean, Long> getCommandHostDelegation(long hostId, Command cmd) {
if (cmd instanceof StorageSubSystemCommand) {

66
server/src/main/java/com/cloud/hypervisor/kvm/dpdk/DPDKHelper.java Normal file
View File

@ -0,0 +1,66 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package com.cloud.hypervisor.kvm.dpdk;
import com.cloud.agent.api.to.VirtualMachineTO;
import com.cloud.vm.VirtualMachineProfile;
import org.apache.cloudstack.api.ApiConstants;
public interface DPDKHelper {
String DPDK_VHOST_USER_MODE = "DPDK-VHOSTUSER";
String DPDK_NUMA = ApiConstants.EXTRA_CONFIG + "-dpdk-numa";
String DPDK_HUGE_PAGES = ApiConstants.EXTRA_CONFIG + "-dpdk-hugepages";
String DPDK_INTERFACE_PREFIX = ApiConstants.EXTRA_CONFIG + "-dpdk-interface-";
enum VHostUserMode {
CLIENT("client"), SERVER("server");
private String str;
VHostUserMode(String str) {
this.str = str;
}
public static VHostUserMode fromValue(String val) {
if (val.equalsIgnoreCase("client")) {
return CLIENT;
} else if (val.equalsIgnoreCase("server")) {
return SERVER;
} else {
throw new IllegalArgumentException("Invalid DPDK vHost User mode:" + val);
}
}
@Override
public String toString() {
return str;
}
}
/**
* True if the DPDK vHost user mode setting is part of the VM service offering details, false if not.
* @param vm
*/
boolean isDPDKvHostUserModeSettingOnServiceOffering(VirtualMachineProfile vm);
/**
* Add DPDK vHost User Mode as extra configuration to the VM TO (if present on the VM service offering details)
*/
void setDpdkVhostUserMode(VirtualMachineTO to, VirtualMachineProfile vm);
}

68
server/src/main/java/com/cloud/hypervisor/kvm/dpdk/DPDKHelperImpl.java Normal file
View File

@ -0,0 +1,68 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package com.cloud.hypervisor.kvm.dpdk;
import com.cloud.agent.api.to.VirtualMachineTO;
import com.cloud.offering.ServiceOffering;
import com.cloud.service.ServiceOfferingDetailsVO;
import com.cloud.service.dao.ServiceOfferingDetailsDao;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.vm.VirtualMachineProfile;
import org.apache.log4j.Logger;
import javax.inject.Inject;
public class DPDKHelperImpl implements DPDKHelper {
@Inject
ServiceOfferingDetailsDao serviceOfferingDetailsDao;
public static final Logger s_logger = Logger.getLogger(DPDKHelperImpl.class);
private ServiceOffering getServiceOfferingFromVMProfile(VirtualMachineProfile virtualMachineProfile) {
ServiceOffering offering = virtualMachineProfile.getServiceOffering();
if (offering == null) {
throw new CloudRuntimeException("VM does not have an associated service offering");
}
return offering;
}
@Override
public boolean isDPDKvHostUserModeSettingOnServiceOffering(VirtualMachineProfile vm) {
ServiceOffering offering = getServiceOfferingFromVMProfile(vm);
ServiceOfferingDetailsVO detail = serviceOfferingDetailsDao.findDetail(offering.getId(), DPDK_VHOST_USER_MODE);
return detail != null;
}
@Override
public void setDpdkVhostUserMode(VirtualMachineTO to, VirtualMachineProfile vm) {
ServiceOffering offering = getServiceOfferingFromVMProfile(vm);
ServiceOfferingDetailsVO detail = serviceOfferingDetailsDao.findDetail(offering.getId(), DPDK_VHOST_USER_MODE);
if (detail != null) {
String mode = detail.getValue();
try {
VHostUserMode dpdKvHostUserMode = VHostUserMode.fromValue(mode);
to.addExtraConfig(DPDK_VHOST_USER_MODE, dpdKvHostUserMode.toString());
} catch (IllegalArgumentException e) {
s_logger.error(String.format("DPDK vHost User mode found as a detail for service offering: %s " +
"but value: %s is not supported. Supported values: %s, %s",
offering.getId(), mode,
VHostUserMode.CLIENT.toString(), VHostUserMode.SERVER.toString()));
}
}
}
}

2
server/src/main/resources/META-INF/cloudstack/core/spring-server-core-misc-context.xml
View File

@ -78,5 +78,7 @@
<bean id="ExternalIpAddressAllocator" class="com.cloud.network.ExternalIpAddressAllocator">
<property name="name" value="Basic" />
</bean>
<bean id="DPDKHelper" class="com.cloud.hypervisor.kvm.dpdk.DPDKHelperImpl" />
</beans>

52
server/src/test/java/com/cloud/hypervisor/KVMGuruTest.java
View File

@ -19,9 +19,13 @@ package com.cloud.hypervisor;
import com.cloud.agent.api.to.VirtualMachineTO;
import com.cloud.host.HostVO;
import com.cloud.host.dao.HostDao;
import com.cloud.offering.ServiceOffering;
import com.cloud.service.ServiceOfferingDetailsVO;
import com.cloud.service.dao.ServiceOfferingDetailsDao;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.vm.VirtualMachine;
import com.cloud.vm.VirtualMachineProfile;
import org.apache.cloudstack.api.ApiConstants;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
@ -31,11 +35,16 @@ import org.mockito.Mockito;
import org.mockito.Spy;
import org.mockito.runners.MockitoJUnitRunner;
import java.io.UnsupportedEncodingException;
import java.util.Arrays;
@RunWith(MockitoJUnitRunner.class)
public class KVMGuruTest {
@Mock
HostDao hostDao;
@Mock
ServiceOfferingDetailsDao serviceOfferingDetailsDao;
@Spy
@InjectMocks
@ -49,18 +58,42 @@ public class KVMGuruTest {
VirtualMachine vm;
@Mock
HostVO host;
@Mock
ServiceOffering serviceOffering;
@Mock
ServiceOfferingDetailsVO detail1;
@Mock
ServiceOfferingDetailsVO detail2;
private static final long hostId = 1l;
private static final long hostId = 1L;
private static final Long offeringId = 1L;
private static final String detail1Key = ApiConstants.EXTRA_CONFIG + "-config-1";
private static final String detail1Value = "value1";
private static final String detail2Key = "detail2";
private static final String detail2Value = "value2";
@Before
public void setup() {
public void setup() throws UnsupportedEncodingException {
Mockito.when(vmTO.getLimitCpuUse()).thenReturn(true);
Mockito.when(vmProfile.getVirtualMachine()).thenReturn(vm);
Mockito.when(vm.getHostId()).thenReturn(hostId);
Mockito.when(hostDao.findById(hostId)).thenReturn(host);
Mockito.when(host.getCpus()).thenReturn(3);
Mockito.when(host.getSpeed()).thenReturn(1995l);
Mockito.when(host.getSpeed()).thenReturn(1995L);
Mockito.when(vmTO.getMaxSpeed()).thenReturn(500);
Mockito.when(serviceOffering.getId()).thenReturn(offeringId);
Mockito.when(vmProfile.getServiceOffering()).thenReturn(serviceOffering);
Mockito.when(detail1.getName()).thenReturn(detail1Key);
Mockito.when(detail1.getValue()).thenReturn(detail1Value);
Mockito.when(detail1.getResourceId()).thenReturn(offeringId);
Mockito.when(detail2.getName()).thenReturn(detail2Key);
Mockito.when(detail2.getResourceId()).thenReturn(offeringId);
Mockito.when(detail2.getValue()).thenReturn(detail2Value);
Mockito.when(serviceOfferingDetailsDao.listDetails(offeringId)).thenReturn(
Arrays.asList(detail1, detail2));
}
@Test
@ -96,4 +129,17 @@ public class KVMGuruTest {
guru.setVmQuotaPercentage(vmTO, vmProfile);
Mockito.verify(vmTO).setCpuQuotaPercentage(1d);
}
@Test
public void testAddServiceOfferingExtraConfigurationDpdkDetails() {
guru.addServiceOfferingExtraConfiguration(vmTO, vmProfile);
Mockito.verify(vmTO).addExtraConfig(detail1Key, detail1Value);
}
@Test
public void testAddServiceOfferingExtraConfigurationEmptyDetails() {
Mockito.when(serviceOfferingDetailsDao.listDetails(offeringId)).thenReturn(null);
guru.addServiceOfferingExtraConfiguration(vmTO, vmProfile);
Mockito.verify(vmTO, Mockito.never()).addExtraConfig(Mockito.anyString(), Mockito.anyString());
}
}

135
server/src/test/java/com/cloud/hypervisor/kvm/dpdk/DPDKHelperImplTest.java Normal file
View File

@ -0,0 +1,135 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package com.cloud.hypervisor.kvm.dpdk;
import com.cloud.agent.api.to.VirtualMachineTO;
import com.cloud.offering.ServiceOffering;
import com.cloud.service.ServiceOfferingDetailsVO;
import com.cloud.service.dao.ServiceOfferingDetailsDao;
import com.cloud.vm.VirtualMachineProfile;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.Spy;
import org.mockito.runners.MockitoJUnitRunner;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.Arrays;
@RunWith(MockitoJUnitRunner.class)
public class DPDKHelperImplTest {
@Mock
ServiceOfferingDetailsDao serviceOfferingDetailsDao;
@Spy
@InjectMocks
private DPDKHelper dpdkHelper = new DPDKHelperImpl();
@Mock
VirtualMachineTO vmTO;
@Mock
VirtualMachineProfile vmProfile;
@Mock
ServiceOfferingDetailsVO dpdkVhostUserModeDetailVO;
@Mock
ServiceOfferingDetailsVO dpdkNumaDetailVO;
@Mock
ServiceOfferingDetailsVO dpdkHugePagesDetailVO;
@Mock
ServiceOffering serviceOffering;
private String dpdkVhostMode = DPDKHelper.VHostUserMode.SERVER.toString();
private static final String dpdkNumaConf =
"<cpu mode=\"host-passthrough\">\n" +
" <numa>\n" +
" <cell id=\"0\" cpus=\"0\" memory=\"9437184\" unit=\"KiB\" memAccess=\"shared\"/>\n" +
" </numa>\n" +
"</cpu>";
private static final String dpdkHugePagesConf =
"<memoryBacking>\n" +
" <hugePages/>\n" +
"</memoryBacking>";
private static String dpdkNumaValue;
private static String dpdkHugePagesValue;
private static final Long offeringId = 1L;
@Before
public void setup() throws UnsupportedEncodingException {
dpdkHugePagesValue = URLEncoder.encode(dpdkHugePagesConf, "UTF-8");
dpdkNumaValue = URLEncoder.encode(dpdkNumaConf, "UTF-8");
Mockito.when(dpdkVhostUserModeDetailVO.getName()).thenReturn(DPDKHelper.DPDK_VHOST_USER_MODE);
Mockito.when(dpdkVhostUserModeDetailVO.getValue()).thenReturn(dpdkVhostMode);
Mockito.when(dpdkVhostUserModeDetailVO.getResourceId()).thenReturn(offeringId);
Mockito.when(dpdkNumaDetailVO.getName()).thenReturn(DPDKHelper.DPDK_NUMA);
Mockito.when(dpdkNumaDetailVO.getResourceId()).thenReturn(offeringId);
Mockito.when(dpdkNumaDetailVO.getValue()).thenReturn(dpdkNumaValue);
Mockito.when(dpdkHugePagesDetailVO.getName()).thenReturn(DPDKHelper.DPDK_HUGE_PAGES);
Mockito.when(dpdkHugePagesDetailVO.getResourceId()).thenReturn(offeringId);
Mockito.when(dpdkHugePagesDetailVO.getValue()).thenReturn(dpdkHugePagesValue);
Mockito.when(serviceOfferingDetailsDao.listDetails(offeringId)).thenReturn(
Arrays.asList(dpdkNumaDetailVO, dpdkHugePagesDetailVO, dpdkVhostUserModeDetailVO));
Mockito.when(vmProfile.getServiceOffering()).thenReturn(serviceOffering);
Mockito.when(serviceOffering.getId()).thenReturn(offeringId);
}
@Test
public void testSetDpdkVhostUserModeValidDetail() {
Mockito.when(serviceOfferingDetailsDao.findDetail(offeringId, DPDKHelper.DPDK_VHOST_USER_MODE)).
thenReturn(dpdkVhostUserModeDetailVO);
dpdkHelper.setDpdkVhostUserMode(vmTO, vmProfile);
Mockito.verify(vmTO).addExtraConfig(DPDKHelper.DPDK_VHOST_USER_MODE, dpdkVhostMode);
}
@Test
public void testSetDpdkVhostUserModeInvalidDetail() {
Mockito.when(dpdkVhostUserModeDetailVO.getValue()).thenReturn("serverrrr");
Mockito.verify(vmTO, Mockito.never()).addExtraConfig(Mockito.anyString(), Mockito.anyString());
}
@Test
public void testSetDpdkVhostUserModeNotExistingDetail() {
Mockito.when(serviceOfferingDetailsDao.listDetails(offeringId)).thenReturn(
Arrays.asList(dpdkNumaDetailVO, dpdkHugePagesDetailVO));
Mockito.verify(vmTO, Mockito.never()).addExtraConfig(Mockito.anyString(), Mockito.anyString());
}
@Test
public void testDPDKvHostUserFromValueClient() {
DPDKHelper.VHostUserMode mode = DPDKHelper.VHostUserMode.fromValue("client");
Assert.assertEquals(DPDKHelper.VHostUserMode.CLIENT, mode);
}
@Test
public void testDPDKvHostUserFromValueServer() {
DPDKHelper.VHostUserMode mode = DPDKHelper.VHostUserMode.fromValue("server");
Assert.assertEquals(DPDKHelper.VHostUserMode.SERVER, mode);
}
@Test(expected = IllegalArgumentException.class)
public void testDPDKvHostUserFromValueServerInvalid() {
DPDKHelper.VHostUserMode.fromValue("serverrrr");
}
}