diff --git a/api/src/main/java/com/cloud/hypervisor/HypervisorGuru.java b/api/src/main/java/com/cloud/hypervisor/HypervisorGuru.java index 96518ac1769..c7dc0bba109 100644 --- a/api/src/main/java/com/cloud/hypervisor/HypervisorGuru.java +++ b/api/src/main/java/com/cloud/hypervisor/HypervisorGuru.java @@ -20,7 +20,6 @@ import java.util.List; import java.util.Map; import org.apache.cloudstack.backup.Backup; -import org.apache.cloudstack.framework.config.ConfigKey; import com.cloud.agent.api.Command; import com.cloud.agent.api.to.NicTO; @@ -35,8 +34,7 @@ import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineProfile; public interface HypervisorGuru extends Adapter { - ConfigKey VmwareFullClone = new ConfigKey("Advanced", Boolean.class, "vmware.create.full.clone", "true", - "If set to true, creates guest VMs as full clones on ESX", false); + HypervisorType getHypervisorType(); /** diff --git a/engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java b/engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java index 44de85edeef..e9a37ae7f46 100644 --- a/engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java +++ b/engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java @@ -40,57 +40,86 @@ public interface CapacityManager { static final String StorageCapacityDisableThresholdCK = "pool.storage.capacity.disablethreshold"; static final String StorageOverprovisioningFactorCK = "storage.overprovisioning.factor"; static final String StorageAllocatedCapacityDisableThresholdCK = "pool.storage.allocated.capacity.disablethreshold"; - static final String VmwareCreateCloneFullCK = "vmware.create.full.clone"; - static final ConfigKey CpuOverprovisioningFactor = new ConfigKey(Float.class, CpuOverprovisioningFactorCK, "Advanced", "1.0", - "Used for CPU overprovisioning calculation; available CPU will be (actualCpuCapacity * cpu.overprovisioning.factor)", true, ConfigKey.Scope.Cluster, null); - static final ConfigKey MemOverprovisioningFactor = new ConfigKey(Float.class, MemOverprovisioningFactorCK, "Advanced", "1.0", - "Used for memory overprovisioning calculation", true, ConfigKey.Scope.Cluster, null); - static final ConfigKey StorageCapacityDisableThreshold = new ConfigKey("Alert", Double.class, StorageCapacityDisableThresholdCK, "0.85", - "Percentage (as a value between 0 and 1) of storage utilization above which allocators will disable using the pool for low storage available.", true, - ConfigKey.Scope.Zone); - static final ConfigKey StorageOverprovisioningFactor = new ConfigKey("Storage", Double.class, StorageOverprovisioningFactorCK, "2", - "Used for storage overprovisioning calculation; available storage will be (actualStorageSize * storage.overprovisioning.factor)", true, ConfigKey.Scope.StoragePool); + static final String CATEGORY_ADVANCED = "Advanced"; + static final String CATEGORY_ALERT = "Alert"; + + static final ConfigKey CpuOverprovisioningFactor = + new ConfigKey<>( + Float.class, + CpuOverprovisioningFactorCK, + CATEGORY_ADVANCED, + "1.0", + "Used for CPU overprovisioning calculation; available CPU will be (actualCpuCapacity * cpu.overprovisioning.factor)", + true, + ConfigKey.Scope.Cluster, + null); + static final ConfigKey MemOverprovisioningFactor = + new ConfigKey<>( + Float.class, + MemOverprovisioningFactorCK, + CATEGORY_ADVANCED, + "1.0", + "Used for memory overprovisioning calculation", + true, + ConfigKey.Scope.Cluster, + null); + static final ConfigKey StorageCapacityDisableThreshold = + new ConfigKey<>( + CATEGORY_ALERT, + Double.class, + StorageCapacityDisableThresholdCK, + "0.85", + "Percentage (as a value between 0 and 1) of storage utilization above which allocators will disable using the pool for low storage available.", + true, + ConfigKey.Scope.Zone); + static final ConfigKey StorageOverprovisioningFactor = + new ConfigKey<>( + "Storage", + Double.class, + StorageOverprovisioningFactorCK, + "2", + "Used for storage overprovisioning calculation; available storage will be (actualStorageSize * storage.overprovisioning.factor)", + true, + ConfigKey.Scope.StoragePool); static final ConfigKey StorageAllocatedCapacityDisableThreshold = - new ConfigKey( - "Alert", - Double.class, - StorageAllocatedCapacityDisableThresholdCK, - "0.85", - "Percentage (as a value between 0 and 1) of allocated storage utilization above which allocators will disable using the pool for low allocated storage available.", - true, ConfigKey.Scope.Zone); + new ConfigKey<>( + CATEGORY_ALERT, + Double.class, + StorageAllocatedCapacityDisableThresholdCK, + "0.85", + "Percentage (as a value between 0 and 1) of allocated storage utilization above which allocators will disable using the pool for low allocated storage available.", + true, + ConfigKey.Scope.Zone); static final ConfigKey StorageOperationsExcludeCluster = - new ConfigKey( + new ConfigKey<>( Boolean.class, "cluster.storage.operations.exclude", - "Advanced", + CATEGORY_ADVANCED, "false", "Exclude cluster from storage operations", true, ConfigKey.Scope.Cluster, null); - static final ConfigKey VmwareCreateCloneFull = - new ConfigKey( - "Storage", - Boolean.class, - VmwareCreateCloneFullCK, - "false", - "If set to true, creates VMs as full clones on ESX hypervisor", - true, - ConfigKey.Scope.StoragePool); static final ConfigKey ImageStoreNFSVersion = - new ConfigKey( + new ConfigKey<>( String.class, "secstorage.nfs.version", - "Advanced", + CATEGORY_ADVANCED, null, "Enforces specific NFS version when mounting Secondary Storage. If NULL default selection is performed", true, ConfigKey.Scope.ImageStore, null); - static final ConfigKey SecondaryStorageCapacityThreshold = new ConfigKey("Advanced", Float.class, "secondary.storage.capacity.threshold", "0.90", - "Percentage (as a value between 0 and 1) of secondary storage capacity threshold.", true); + static final ConfigKey SecondaryStorageCapacityThreshold = + new ConfigKey<>( + CATEGORY_ADVANCED, + Float.class, + "secondary.storage.capacity.threshold", + "0.90", + "Percentage (as a value between 0 and 1) of secondary storage capacity threshold.", + true); public boolean releaseVmCapacity(VirtualMachine vm, boolean moveFromReserved, boolean moveToReservered, Long hostId); diff --git a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java index 1b69d723000..130b1607764 100644 --- a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java +++ b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java @@ -163,6 +163,44 @@ public interface StorageManager extends StorageService { true, ConfigKey.Scope.Cluster, null); + ConfigKey VmwareCreateCloneFull = new ConfigKey<>(Boolean.class, + "vmware.create.full.clone", + "Storage", + "false", + "If set to true, creates VMs as full clones on ESX hypervisor", + true, + ConfigKey.Scope.StoragePool, + null); + ConfigKey VmwareAllowParallelExecution = new ConfigKey<>(Boolean.class, + "vmware.allow.parallel.command.execution", + "Advanced", + "false", + "allow commands to be executed in parallel in spite of 'vmware.create.full.clone' being set to true.", + true, + ConfigKey.Scope.Global, + null); + + /** + * should we execute in sequence not involving any storages? + * @return tru if commands should execute in sequence + */ + static boolean shouldExecuteInSequenceOnVmware() { + return shouldExecuteInSequenceOnVmware(null, null); + } + + static boolean shouldExecuteInSequenceOnVmware(Long srcStoreId, Long dstStoreId) { + final Boolean fullClone = getFullCloneConfiguration(srcStoreId) || getFullCloneConfiguration(dstStoreId); + final Boolean allowParallel = getAllowParallelExecutionConfiguration(); + return fullClone && !allowParallel; + } + + static Boolean getAllowParallelExecutionConfiguration() { + return VmwareAllowParallelExecution.value(); + } + + static Boolean getFullCloneConfiguration(Long storeId) { + return VmwareCreateCloneFull.valueIn(storeId); + } /** * Returns a comma separated list of tags for the specified storage pool diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 64e4ad89154..a1831a5cacd 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -1557,8 +1557,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac case LXC: return false; case VMware: - final Boolean fullClone = HypervisorGuru.VmwareFullClone.value(); - return fullClone; + return StorageManager.shouldExecuteInSequenceOnVmware(); default: return ExecuteInSequence.value(); } diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java index 476ecf23cbc..4cd4d786c74 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java @@ -85,7 +85,6 @@ import com.cloud.agent.api.to.DatadiskTO; import com.cloud.agent.api.to.DiskTO; import com.cloud.agent.api.to.VirtualMachineTO; import com.cloud.agent.manager.allocator.PodAllocator; -import com.cloud.capacity.CapacityManager; import com.cloud.cluster.ClusterManager; import com.cloud.configuration.Resource.ResourceType; import com.cloud.dc.DataCenter; @@ -1704,7 +1703,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati if (vm.getHypervisorType().equals(HypervisorType.VMware)) { // retrieve clone flag. UserVmCloneType cloneType = UserVmCloneType.linked; - Boolean value = CapacityManager.VmwareCreateCloneFull.valueIn(vol.getPoolId()); + Boolean value = StorageManager.VmwareCreateCloneFull.valueIn(vol.getPoolId()); if (value != null && value) { cloneType = UserVmCloneType.full; } diff --git a/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java b/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java index 86b0ea4a96d..cb3a6b77b88 100644 --- a/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java +++ b/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java @@ -32,6 +32,7 @@ import java.util.List; import java.util.Map; import com.cloud.exception.InvalidParameterValueException; +import com.cloud.storage.StorageManager; import org.apache.cloudstack.engine.subsystem.api.storage.StoragePoolAllocator; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; @@ -56,7 +57,6 @@ import com.cloud.deploy.DeploymentPlanner; import com.cloud.deploy.DeploymentPlanner.ExcludeList; import com.cloud.host.HostVO; import com.cloud.hypervisor.Hypervisor.HypervisorType; -import com.cloud.hypervisor.HypervisorGuru; import com.cloud.service.ServiceOfferingVO; import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.storage.DiskOfferingVO; @@ -209,9 +209,23 @@ public class VirtualMachineManagerImplTest { public void testExeceuteInSequence() { assertTrue(virtualMachineManagerImpl.getExecuteInSequence(HypervisorType.XenServer) == false); assertTrue(virtualMachineManagerImpl.getExecuteInSequence(HypervisorType.KVM) == false); - assertTrue(virtualMachineManagerImpl.getExecuteInSequence(HypervisorType.VMware) == HypervisorGuru.VmwareFullClone.value()); assertTrue(virtualMachineManagerImpl.getExecuteInSequence(HypervisorType.Ovm3) == VirtualMachineManager.ExecuteInSequence.value()); } + @Test + public void testExeceuteInSequenceVmware() { + when(StorageManager.getFullCloneConfiguration(anyLong())).thenReturn(Boolean.FALSE); + when(StorageManager.getAllowParallelExecutionConfiguration()).thenReturn(Boolean.FALSE); + assertFalse("no full clones so no need to execute in sequence", virtualMachineManagerImpl.getExecuteInSequence(HypervisorType.VMware)); + when(StorageManager.getFullCloneConfiguration(anyLong())).thenReturn(Boolean.TRUE); + when(StorageManager.getAllowParallelExecutionConfiguration()).thenReturn(Boolean.FALSE); + assertTrue("full clones and no explicit parallel execution allowed, should execute in sequence", virtualMachineManagerImpl.getExecuteInSequence(HypervisorType.VMware)); + when(StorageManager.getFullCloneConfiguration(anyLong())).thenReturn(Boolean.TRUE); + when(StorageManager.getAllowParallelExecutionConfiguration()).thenReturn(Boolean.TRUE); + assertFalse("execute in sequence should not be needed as parallel is allowed", virtualMachineManagerImpl.getExecuteInSequence(HypervisorType.VMware)); + when(StorageManager.getFullCloneConfiguration(anyLong())).thenReturn(Boolean.FALSE); + when(StorageManager.getAllowParallelExecutionConfiguration()).thenReturn(Boolean.TRUE); + assertFalse("double reasons to allow parallel execution", virtualMachineManagerImpl.getExecuteInSequence(HypervisorType.VMware)); + } @Test public void testCheckIfCanUpgrade() throws Exception { diff --git a/engine/schema/src/main/java/com/cloud/network/dao/NetworkDao.java b/engine/schema/src/main/java/com/cloud/network/dao/NetworkDao.java index dfd2a046bb7..0861a424ebb 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/NetworkDao.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/NetworkDao.java @@ -47,10 +47,6 @@ public interface NetworkDao extends GenericDao, StateDao getCommandHostDelegation(long hostId, Command cmd) { + if (s_logger.isTraceEnabled()) { + s_logger.trace(String.format("Finding delegation for command of type %s to host %d.", cmd.getClass(), hostId)); + } + boolean needDelegation = false; if (cmd instanceof StorageSubSystemCommand) { - Boolean fullCloneEnabled = VmwareFullClone.value(); StorageSubSystemCommand c = (StorageSubSystemCommand)cmd; - c.setExecuteInSequence(fullCloneEnabled); + c.setExecuteInSequence(StorageManager.shouldExecuteInSequenceOnVmware()); } if (cmd instanceof DownloadCommand) { cmd.setContextParam(VmwareManager.s_vmwareOVAPackageTimeout.key(), String.valueOf(VmwareManager.s_vmwareOVAPackageTimeout.value())); @@ -234,11 +227,7 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co DataStoreTO destStoreTO = destData.getDataStore(); boolean inSeq = true; - if ((srcData.getObjectType() == DataObjectType.SNAPSHOT) || (destData.getObjectType() == DataObjectType.SNAPSHOT)) { - inSeq = false; - } else if ((destStoreTO.getRole() == DataStoreRole.Image) || (destStoreTO.getRole() == DataStoreRole.ImageCache)) { - inSeq = false; - } else if (!VmwareFullClone.value()) { + if (parallelExecutionAllowed(srcData, destData, srcStoreTO, destStoreTO)) { inSeq = false; } cpyCommand.setExecuteInSequence(inSeq); @@ -273,10 +262,15 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co } } + if (s_logger.isTraceEnabled()) { + s_logger.trace(String.format("Command of type %s is going to be executed in sequence? %b", cmd.getClass(), cmd.executeInSequence())); + s_logger.trace(String.format("Command of type %s is going to need delegation? %b", cmd.getClass(), needDelegation)); + } + if (!needDelegation) { return new Pair(Boolean.FALSE, new Long(hostId)); } - HostVO host = _hostDao.findById(hostId); + HostVO host = hostDao.findById(hostId); long dcId = host.getDataCenterId(); Pair cmdTarget = _secStorageMgr.assignSecStorageVm(dcId, cmd); if (cmdTarget != null) { @@ -322,6 +316,16 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co return new Pair(Boolean.FALSE, new Long(hostId)); } + private boolean parallelExecutionAllowed(DataTO srcData, DataTO destData, DataStoreTO srcStoreTO, DataStoreTO destStoreTO) { + Long srcId = srcStoreTO == null || srcStoreTO.getUuid() == null ? null : _storagePoolDao.findByUuid(srcStoreTO.getUuid()).getId(); + Long dstId = destStoreTO == null || destStoreTO.getUuid() == null ? null : _storagePoolDao.findByUuid(destStoreTO.getUuid()).getId(); + return (srcData.getObjectType() == DataObjectType.SNAPSHOT) + || (destData.getObjectType() == DataObjectType.SNAPSHOT) + || (destStoreTO != null && destStoreTO.getRole() == DataStoreRole.Image) + || (destStoreTO != null && destStoreTO.getRole() == DataStoreRole.ImageCache) + || !StorageManager.shouldExecuteInSequenceOnVmware(srcId, dstId); + } + @Override public boolean trackVmHostChange() { return true; @@ -345,12 +349,12 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co @Override public List finalizeExpungeNics(VirtualMachine vm, List nics) { List commands = new ArrayList(); - List nicVOs = _nicDao.listByVmId(vm.getId()); + List nicVOs = nicDao.listByVmId(vm.getId()); for (NicVO nic : nicVOs) { - NetworkVO network = _networkDao.findById(nic.getNetworkId()); + NetworkVO network = networkDao.findById(nic.getNetworkId()); if (network.getBroadcastDomainType() == BroadcastDomainType.Lswitch) { s_logger.debug("Nic " + nic.toString() + " is connected to an lswitch, cleanup required"); - NetworkVO networkVO = _networkDao.findById(nic.getNetworkId()); + NetworkVO networkVO = networkDao.findById(nic.getNetworkId()); // We need the traffic label to figure out which vSwitch has the // portgroup PhysicalNetworkTrafficTypeVO trafficTypeVO = _physicalNetworkTrafficTypeDao.findBy(networkVO.getPhysicalNetworkId(), networkVO.getTrafficType()); @@ -488,7 +492,7 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co } VolumeVO volumeVO = disksMapping.get(disk); if (volumeVO == null) { - final VMInstanceVO vm = _vmDao.findByIdIncludingRemoved(backup.getVmId()); + final VMInstanceVO vm = virtualMachineDao.findByIdIncludingRemoved(backup.getVmId()); if (vm == null) { throw new CloudRuntimeException("Failed to find the volumes details from the VM backup"); } @@ -655,18 +659,18 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co private VMInstanceVO getVM(String vmInternalName, long templateId, long guestOsId, long serviceOfferingId, long zoneId, long accountId, long userId, long domainId) { s_logger.debug(String.format("Trying to get VM with specs: [vmInternalName: %s, templateId: %s, guestOsId: %s, serviceOfferingId: %s].", vmInternalName, templateId, guestOsId, serviceOfferingId)); - VMInstanceVO vm = _vmDao.findVMByInstanceNameIncludingRemoved(vmInternalName); + VMInstanceVO vm = virtualMachineDao.findVMByInstanceNameIncludingRemoved(vmInternalName); if (vm != null) { s_logger.debug(String.format("Found an existing VM [id: %s, removed: %s] with internalName: [%s].", vm.getUuid(), vm.getRemoved() != null ? "yes" : "no", vmInternalName)); vm.setState(VirtualMachine.State.Stopped); vm.setPowerState(VirtualMachine.PowerState.PowerOff); - _vmDao.update(vm.getId(), vm); + virtualMachineDao.update(vm.getId(), vm); if (vm.getRemoved() != null) { - _vmDao.unremove(vm.getId()); + virtualMachineDao.unremove(vm.getId()); UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_CREATE, accountId, vm.getDataCenterId(), vm.getId(), vm.getHostName(), vm.getServiceOfferingId(), vm.getTemplateId(), vm.getHypervisorType().toString(), VirtualMachine.class.getName(), vm.getUuid(), vm.isDisplayVm()); } - return _vmDao.findById(vm.getId()); + return virtualMachineDao.findById(vm.getId()); } else { long id = userVmDao.getNextInSequence(Long.class, "id"); s_logger.debug(String.format("Can't find an existing VM with internalName: [%s]. Creating a new VM with: [id: %s, name: %s, templateId: %s, guestOsId: %s, serviceOfferingId: %s].", @@ -783,7 +787,7 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co } private VolumeVO createVolume(VirtualDisk disk, VirtualMachineMO vmToImport, long domainId, long zoneId, long accountId, long instanceId, Long poolId, long templateId, Backup backup, boolean isImport) throws Exception { - VMInstanceVO vm = _vmDao.findByIdIncludingRemoved(backup.getVmId()); + VMInstanceVO vm = virtualMachineDao.findByIdIncludingRemoved(backup.getVmId()); if (vm == null) { throw new CloudRuntimeException("Failed to find the backup volume information from the VM backup"); } @@ -832,13 +836,13 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co */ private NetworkVO createNetworkRecord(Long zoneId, String tag, String vlan, long accountId, long domainId) { Long physicalNetworkId = getPhysicalNetworkId(zoneId, tag); - final long id = _networkDao.getNextInSequence(Long.class, "id"); + final long id = networkDao.getNextInSequence(Long.class, "id"); NetworkVO networkVO = new NetworkVO(id, TrafficType.Guest, Networks.Mode.Dhcp, BroadcastDomainType.Vlan, 9L, domainId, accountId, id, "Imported-network-" + id, "Imported-network-" + id, null, Network.GuestType.Isolated, zoneId, physicalNetworkId, ControlledEntity.ACLType.Account, false, null, false); networkVO.setBroadcastUri(BroadcastDomainType.Vlan.toUri(vlan)); networkVO.setGuruName("ExternalGuestNetworkGuru"); networkVO.setState(Network.State.Implemented); - return _networkDao.persist(networkVO); + return networkDao.persist(networkVO); } /** @@ -852,7 +856,7 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co String tag = parts[parts.length - 1]; String[] tagSplit = tag.split("-"); tag = tagSplit[tagSplit.length - 1]; - NetworkVO networkVO = _networkDao.findByVlan(vlan); + NetworkVO networkVO = networkDao.findByVlan(vlan); if (networkVO == null) { networkVO = createNetworkRecord(zoneId, tag, vlan, accountId, domainId); } @@ -894,13 +898,13 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co private void syncVMNics(VirtualDevice[] nicDevices, DatacenterMO dcMo, Map networksMapping, VMInstanceVO vm) throws Exception { VmwareContext context = dcMo.getContext(); - List allNics = _nicDao.listByVmId(vm.getId()); + List allNics = nicDao.listByVmId(vm.getId()); for (VirtualDevice nicDevice : nicDevices) { Pair pair = getNicMacAddressAndNetworkName(nicDevice, context); String macAddress = pair.first(); String networkName = pair.second(); NetworkVO networkVO = networksMapping.get(networkName); - NicVO nicVO = _nicDao.findByNetworkIdAndMacAddress(networkVO.getId(), macAddress); + NicVO nicVO = nicDao.findByNetworkIdAndMacAddress(networkVO.getId(), macAddress); if (nicVO != null) { allNics.remove(nicVO); } @@ -911,7 +915,7 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co } private Map getDisksMapping(Backup backup, List virtualDisks) { - final VMInstanceVO vm = _vmDao.findByIdIncludingRemoved(backup.getVmId()); + final VMInstanceVO vm = virtualMachineDao.findByIdIncludingRemoved(backup.getVmId()); if (vm == null) { throw new CloudRuntimeException("Failed to find the volumes details from the VM backup"); } @@ -1076,7 +1080,7 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co private String getHostGuidForLocalStorage(StoragePool pool) { List storagePoolHostVOs = storagePoolHostDao.listByPoolId(pool.getId()); StoragePoolHostVO storagePoolHostVO = storagePoolHostVOs.get(0); - HostVO hostVO = _hostDao.findById(storagePoolHostVO.getHostId()); + HostVO hostVO = hostDao.findById(storagePoolHostVO.getHostId()); return hostVO.getGuid(); } @@ -1087,7 +1091,7 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co // Without host vMotion might fail between non-shared storages with error similar to, // https://kb.vmware.com/s/article/1003795 // As this is offline migration VM won't be started on this host - List hosts = _hostDao.findHypervisorHostInCluster(destClusterId); + List hosts = hostDao.findHypervisorHostInCluster(destClusterId); if (CollectionUtils.isNotEmpty(hosts)) { hostInTargetCluster = hosts.get(0); } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/CleanupFullyClonedTemplatesTask.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/CleanupFullyClonedTemplatesTask.java index 41bbcb027ae..1437e057b86 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/CleanupFullyClonedTemplatesTask.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/CleanupFullyClonedTemplatesTask.java @@ -19,7 +19,7 @@ package com.cloud.hypervisor.vmware.manager; import com.cloud.api.query.dao.TemplateJoinDao; import com.cloud.api.query.vo.TemplateJoinVO; import com.cloud.hypervisor.Hypervisor; -import com.cloud.hypervisor.HypervisorGuru; +import com.cloud.storage.StorageManager; import com.cloud.storage.VMTemplateStoragePoolVO; import com.cloud.storage.dao.VMTemplatePoolDao; import com.cloud.template.TemplateManager; @@ -74,7 +74,7 @@ public class CleanupFullyClonedTemplatesTask extends ManagedContextRunnable { mine = Thread.currentThread(); s_logger.info("running job to mark fully cloned templates for gc in thread " + mine.getName()); - if (HypervisorGuru.VmwareFullClone.value()) { // only run if full cloning is being used (might need to be more fine grained) + if (StorageManager.VmwareCreateCloneFull.value()) { // only run if full cloning is being used (might need to be more fine grained) try { queryAllPools(); } catch (Throwable t) { diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java index 44702dca116..d9a01203a03 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java @@ -331,12 +331,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw _storage.configure("StorageLayer", params); } - value = _configDao.getValue(Config.VmwareCreateFullClone.key()); - if (value == null) { - _fullCloneFlag = false; - } else { - _fullCloneFlag = Boolean.parseBoolean(value); - } + _fullCloneFlag = StorageManager.VmwareCreateCloneFull.value(); value = _configDao.getValue(Config.SetVmInternalNameUsingDisplayName.key()); if (value == null) { diff --git a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/XenServerGuru.java b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/XenServerGuru.java index 65eba99ba4e..a8d7a6e268e 100644 --- a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/XenServerGuru.java +++ b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/XenServerGuru.java @@ -44,7 +44,6 @@ import com.cloud.agent.api.to.DiskTO; import com.cloud.agent.api.to.NfsTO; import com.cloud.agent.api.to.VirtualMachineTO; import com.cloud.host.HostVO; -import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.storage.GuestOSHypervisorVO; import com.cloud.storage.GuestOSVO; @@ -68,8 +67,6 @@ public class XenServerGuru extends HypervisorGuruBase implements HypervisorGuru, @Inject private GuestOSHypervisorDao guestOsHypervisorDao; @Inject - private HostDao hostDao; - @Inject private VolumeDao volumeDao; @Inject private PrimaryDataStoreDao storagePoolDao; @@ -178,10 +175,6 @@ public class XenServerGuru extends HypervisorGuruBase implements HypervisorGuru, @Override public Pair getCommandHostDelegation(long hostId, Command cmd) { - if (cmd instanceof StorageSubSystemCommand) { - StorageSubSystemCommand c = (StorageSubSystemCommand)cmd; - c.setExecuteInSequence(true); - } boolean isCopyCommand = cmd instanceof CopyCommand; Pair defaultHostToExecuteCommands = super.getCommandHostDelegation(hostId, cmd); if (!isCopyCommand) { @@ -197,6 +190,14 @@ public class XenServerGuru extends HypervisorGuruBase implements HypervisorGuru, logger.debug("We are returning the default host to execute commands because the target hypervisor of the source data is not XenServer."); return defaultHostToExecuteCommands; } + // only now can we decide, now we now we're only deciding for ourselves + if (cmd instanceof StorageSubSystemCommand) { + if (s_logger.isTraceEnabled()) { + s_logger.trace(String.format("XenServer StrorageSubSystemCommand re always executed in sequence (command of type %s to host %l).", cmd.getClass(), hostId)); + } + StorageSubSystemCommand c = (StorageSubSystemCommand)cmd; + c.setExecuteInSequence(true); + } DataStoreTO srcStore = srcData.getDataStore(); DataStoreTO destStore = destData.getDataStore(); boolean isSourceAndDestinationNfsObjects = srcStore instanceof NfsTO && destStore instanceof NfsTO; diff --git a/plugins/hypervisors/xenserver/src/test/java/com/cloud/hypervisor/XenServerGuruTest.java b/plugins/hypervisors/xenserver/src/test/java/com/cloud/hypervisor/XenServerGuruTest.java index cc6621452c1..f5169ae5dde 100644 --- a/plugins/hypervisors/xenserver/src/test/java/com/cloud/hypervisor/XenServerGuruTest.java +++ b/plugins/hypervisors/xenserver/src/test/java/com/cloud/hypervisor/XenServerGuruTest.java @@ -19,7 +19,6 @@ package com.cloud.hypervisor; import org.apache.cloudstack.hypervisor.xenserver.XenserverConfigs; import org.apache.cloudstack.storage.command.CopyCommand; -import org.apache.cloudstack.storage.command.StorageSubSystemCommand; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.commons.lang3.StringUtils; import org.junit.Assert; @@ -41,6 +40,8 @@ import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.utils.Pair; +import static org.mockito.Mockito.times; + @RunWith(MockitoJUnitRunner.class) public class XenServerGuruTest { @@ -102,13 +103,33 @@ public class XenServerGuruTest { } @Test - public void getCommandHostDelegationTestCommanIsStorageSubSystemCommand() { - StorageSubSystemCommand storageSubSystemCommandMock = Mockito.mock(StorageSubSystemCommand.class); - Pair pairHostToExecuteCommand = xenServerGuru.getCommandHostDelegation(defaultHostId, storageSubSystemCommandMock); + public void getCommandHostDelegationTestCommandIsStorageSubSystemCommand() { + CopyCommand copyCommand = Mockito.mock(CopyCommand.class); + DataTO srcData = Mockito.mock(DataTO.class); + DataTO dstData = Mockito.mock(DataTO.class); + Mockito.when(copyCommand.getSrcTO()).thenReturn(srcData); + Mockito.when(copyCommand.getDestTO()).thenReturn(dstData); + Mockito.when(srcData.getHypervisorType()).thenReturn(HypervisorType.XenServer); + + Pair pairHostToExecuteCommand = xenServerGuru.getCommandHostDelegation(defaultHostId, copyCommand); assertPairOfHostToExecuteCommandIsTheDefaultHostId(pairHostToExecuteCommand); - Mockito.verify(storageSubSystemCommandMock).setExecuteInSequence(true); + Mockito.verify(copyCommand).setExecuteInSequence(true); + } + + @Test + public void getCommandHostDelegationTestCommandIsNotForXenHypervisor() { + CopyCommand copyCommand = Mockito.mock(CopyCommand.class); + DataTO srcData = Mockito.mock(DataTO.class); + Mockito.when(copyCommand.getSrcTO()).thenReturn(srcData); + Mockito.when(srcData.getHypervisorType()).thenReturn(HypervisorType.Any); + + Pair pairHostToExecuteCommand = xenServerGuru.getCommandHostDelegation(defaultHostId, copyCommand); + + assertPairOfHostToExecuteCommandIsTheDefaultHostId(pairHostToExecuteCommand); + + Mockito.verify(copyCommand, times(0)).setExecuteInSequence(true); } @Test diff --git a/server/src/main/java/com/cloud/capacity/CapacityManagerImpl.java b/server/src/main/java/com/cloud/capacity/CapacityManagerImpl.java index a6aa9fb3bb2..726161740c1 100644 --- a/server/src/main/java/com/cloud/capacity/CapacityManagerImpl.java +++ b/server/src/main/java/com/cloud/capacity/CapacityManagerImpl.java @@ -1256,6 +1256,6 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager, @Override public ConfigKey[] getConfigKeys() { return new ConfigKey[] {CpuOverprovisioningFactor, MemOverprovisioningFactor, StorageCapacityDisableThreshold, StorageOverprovisioningFactor, - StorageAllocatedCapacityDisableThreshold, StorageOperationsExcludeCluster, VmwareCreateCloneFull, ImageStoreNFSVersion, SecondaryStorageCapacityThreshold}; + StorageAllocatedCapacityDisableThreshold, StorageOperationsExcludeCluster, ImageStoreNFSVersion, SecondaryStorageCapacityThreshold}; } } diff --git a/server/src/main/java/com/cloud/configuration/Config.java b/server/src/main/java/com/cloud/configuration/Config.java index 1fedad43068..67fe9ad19a8 100644 --- a/server/src/main/java/com/cloud/configuration/Config.java +++ b/server/src/main/java/com/cloud/configuration/Config.java @@ -1087,14 +1087,6 @@ public enum Config { "false", "Enable/Disable Nexus/Vmware dvSwitch in VMware environment", null), - VmwareCreateFullClone( - "Advanced", - ManagementServer.class, - Boolean.class, - "vmware.create.full.clone", - "true", - "If set to true, creates guest VMs as full clones on ESX", - null), VmwareServiceConsole( "Advanced", ManagementServer.class, diff --git a/server/src/main/java/com/cloud/hypervisor/HypervisorGuruBase.java b/server/src/main/java/com/cloud/hypervisor/HypervisorGuruBase.java index fe37e670306..20b7a2b67d6 100644 --- a/server/src/main/java/com/cloud/hypervisor/HypervisorGuruBase.java +++ b/server/src/main/java/com/cloud/hypervisor/HypervisorGuruBase.java @@ -71,13 +71,16 @@ public abstract class HypervisorGuruBase extends AdapterBase implements Hypervis public static final Logger s_logger = Logger.getLogger(HypervisorGuruBase.class); @Inject - private NicDao _nicDao; + protected + NicDao nicDao; @Inject - private NetworkDao _networkDao; + protected + NetworkDao networkDao; @Inject private NetworkOfferingDetailsDao networkOfferingDetailsDao; @Inject - private VMInstanceDao _virtualMachineDao; + protected + VMInstanceDao virtualMachineDao; @Inject private UserVmDetailsDao _userVmDetailsDao; @Inject @@ -87,11 +90,12 @@ public abstract class HypervisorGuruBase extends AdapterBase implements Hypervis @Inject protected ServiceOfferingDetailsDao _serviceOfferingDetailsDao; @Inject - private ServiceOfferingDao _serviceOfferingDao; + protected ServiceOfferingDao serviceOfferingDao; @Inject private NetworkDetailsDao networkDetailsDao; @Inject - private HostDao hostDao; + protected + HostDao hostDao; public static ConfigKey VmMinMemoryEqualsMemoryDividedByMemOverprovisioningFactor = new ConfigKey("Advanced", Boolean.class, "vm.min.memory.equals.memory.divided.by.mem.overprovisioning.factor", "true", "If we set this to 'true', a minimum memory (memory/ mem.overprovisioning.factor) will be set to the VM, independent of using a scalable service offering or not.", true, ConfigKey.Scope.Cluster); @@ -140,11 +144,11 @@ public abstract class HypervisorGuruBase extends AdapterBase implements Hypervis to.setIp6Gateway(profile.getIPv6Gateway()); to.setIp6Cidr(profile.getIPv6Cidr()); - NetworkVO network = _networkDao.findById(profile.getNetworkId()); + NetworkVO network = networkDao.findById(profile.getNetworkId()); to.setNetworkUuid(network.getUuid()); // Workaround to make sure the TO has the UUID we need for Nicira integration - NicVO nicVO = _nicDao.findById(profile.getId()); + NicVO nicVO = nicDao.findById(profile.getId()); if (nicVO != null) { to.setUuid(nicVO.getUuid()); // disable pxe on system vm nics to speed up boot time @@ -199,7 +203,7 @@ public abstract class HypervisorGuruBase extends AdapterBase implements Hypervis } protected VirtualMachineTO toVirtualMachineTO(VirtualMachineProfile vmProfile) { - ServiceOffering offering = _serviceOfferingDao.findById(vmProfile.getId(), vmProfile.getServiceOfferingId()); + ServiceOffering offering = serviceOfferingDao.findById(vmProfile.getId(), vmProfile.getServiceOfferingId()); VirtualMachine vm = vmProfile.getVirtualMachine(); Long clusterId = findClusterOfVm(vm); boolean divideMemoryByOverprovisioning = true; @@ -268,7 +272,7 @@ public abstract class HypervisorGuruBase extends AdapterBase implements Hypervis } // Workaround to make sure the TO has the UUID we need for Niciri integration - VMInstanceVO vmInstance = _virtualMachineDao.findById(to.getId()); + VMInstanceVO vmInstance = virtualMachineDao.findById(to.getId()); to.setEnableDynamicallyScaleVm(vmInstance.isDynamicallyScalable()); to.setUuid(vmInstance.getUuid()); diff --git a/server/src/main/java/com/cloud/hypervisor/KVMGuru.java b/server/src/main/java/com/cloud/hypervisor/KVMGuru.java index c18f39a3a6b..b69953c9ed3 100644 --- a/server/src/main/java/com/cloud/hypervisor/KVMGuru.java +++ b/server/src/main/java/com/cloud/hypervisor/KVMGuru.java @@ -22,11 +22,9 @@ import com.cloud.agent.api.to.NicTO; import com.cloud.agent.api.to.VirtualMachineTO; import com.cloud.configuration.ConfigurationManagerImpl; import com.cloud.host.HostVO; -import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.hypervisor.kvm.dpdk.DpdkHelper; import com.cloud.service.ServiceOfferingVO; -import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.storage.DataStoreRole; import com.cloud.storage.GuestOSHypervisorVO; import com.cloud.storage.GuestOSVO; @@ -56,12 +54,8 @@ public class KVMGuru extends HypervisorGuruBase implements HypervisorGuru { @Inject GuestOSHypervisorDao _guestOsHypervisorDao; @Inject - HostDao _hostDao; - @Inject DpdkHelper dpdkHelper; - @Inject - ServiceOfferingDao serviceOfferingDao; public static final Logger s_logger = Logger.getLogger(KVMGuru.class); @@ -93,7 +87,7 @@ public class KVMGuru extends HypervisorGuruBase implements HypervisorGuru { protected void setVmQuotaPercentage(VirtualMachineTO to, VirtualMachineProfile vmProfile) { if (to.getLimitCpuUse()) { VirtualMachine vm = vmProfile.getVirtualMachine(); - HostVO host = _hostDao.findById(vm.getHostId()); + HostVO host = hostDao.findById(vm.getHostId()); if (host == null) { throw new CloudRuntimeException("Host with id: " + vm.getHostId() + " not found"); } @@ -126,7 +120,7 @@ public class KVMGuru extends HypervisorGuruBase implements HypervisorGuru { VirtualMachine virtualMachine = vm.getVirtualMachine(); Long hostId = virtualMachine.getHostId(); - HostVO host = hostId == null ? null : _hostDao.findById(hostId); + HostVO host = hostId == null ? null : hostDao.findById(hostId); // Determine the VM's OS description configureVmOsDescription(virtualMachine, to, host); @@ -206,7 +200,7 @@ public class KVMGuru extends HypervisorGuruBase implements HypervisorGuru { Long lastHostId = virtualMachine.getLastHostId(); s_logger.info(String.format("%s is not running; therefore, we use the last host [%s] that the VM was running on to derive the unconstrained service offering max CPU and memory.", vmDescription, lastHostId)); - HostVO lastHost = lastHostId == null ? null : _hostDao.findById(lastHostId); + HostVO lastHost = lastHostId == null ? null : hostDao.findById(lastHostId); if (lastHost != null) { maxHostMemory = lastHost.getTotalMemory(); maxHostCpuCore = lastHost.getCpus(); diff --git a/server/src/main/java/com/cloud/hypervisor/LXCGuru.java b/server/src/main/java/com/cloud/hypervisor/LXCGuru.java index 285f9628d8d..7f155a7c17b 100644 --- a/server/src/main/java/com/cloud/hypervisor/LXCGuru.java +++ b/server/src/main/java/com/cloud/hypervisor/LXCGuru.java @@ -22,7 +22,6 @@ import javax.inject.Inject; import com.cloud.agent.api.to.VirtualMachineTO; import com.cloud.host.HostVO; -import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.storage.GuestOSHypervisorVO; import com.cloud.storage.GuestOSVO; @@ -35,8 +34,6 @@ public class LXCGuru extends HypervisorGuruBase implements HypervisorGuru { GuestOSDao _guestOsDao; @Inject GuestOSHypervisorDao _guestOsHypervisorDao; - @Inject - HostDao _hostDao; @Override public HypervisorType getHypervisorType() { @@ -55,7 +52,7 @@ public class LXCGuru extends HypervisorGuruBase implements HypervisorGuru { GuestOSVO guestOS = _guestOsDao.findByIdIncludingRemoved(vm.getVirtualMachine().getGuestOSId()); to.setOs(guestOS.getDisplayName()); - HostVO host = _hostDao.findById(vm.getVirtualMachine().getHostId()); + HostVO host = hostDao.findById(vm.getVirtualMachine().getHostId()); GuestOSHypervisorVO guestOsMapping = null; if (host != null) { guestOsMapping = _guestOsHypervisorDao.findByOsIdAndHypervisor(guestOS.getId(), getHypervisorType().toString(), host.getHypervisorVersion()); diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index de4eb55bc92..140f62d14b8 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -3374,7 +3374,9 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C DiskProvisioningStrictness, PreferredStoragePool, SecStorageVMAutoScaleDown, - MountDisabledStoragePool + MountDisabledStoragePool, + VmwareCreateCloneFull, + VmwareAllowParallelExecution }; } diff --git a/server/src/test/java/com/cloud/hypervisor/KVMGuruTest.java b/server/src/test/java/com/cloud/hypervisor/KVMGuruTest.java index 1f88173df2b..0999cd03909 100644 --- a/server/src/test/java/com/cloud/hypervisor/KVMGuruTest.java +++ b/server/src/test/java/com/cloud/hypervisor/KVMGuruTest.java @@ -274,7 +274,7 @@ public class KVMGuruTest { public void validateGetHostMaxMemoryAndCpuCoresHostNullAndLastHostIdNotNullAndLastHostNull(){ Long maxMemory = Long.MAX_VALUE; Integer maxCpuCores = Integer.MAX_VALUE; - guru._hostDao = hostDao; + guru.hostDao = hostDao; Mockito.when(virtualMachineMock.getLastHostId()).thenReturn(1l); Mockito.doReturn(null).when(hostDao).findById(Mockito.any()); @@ -288,7 +288,7 @@ public class KVMGuruTest { public void validateGetHostMaxMemoryAndCpuCoresHostNullAndLastHostIdNotNullAndLastHostNotNull(){ Long maxMemory = 2048l; Integer maxCpuCores = 16; - guru._hostDao = hostDao; + guru.hostDao = hostDao; Mockito.when(virtualMachineMock.getLastHostId()).thenReturn(1l); Mockito.doReturn(host).when(hostDao).findById(Mockito.any());