From 5820b071b8e1163d46053e686ae4be54f09021d8 Mon Sep 17 00:00:00 2001 From: Kelven Yang Date: Fri, 13 Sep 2013 16:26:40 -0700 Subject: [PATCH] CLOUDSTACK-4659: Add the missing feature back for GC VMware worker VMs --- .../consoleproxy/ConsoleProxyResource.java | 25 +-- client/pom.xml | 2 +- .../src/com/cloud/cluster/ClusterManager.java | 2 + .../com/cloud/cluster/ClusterManagerImpl.java | 4 + .../com/cloud/hypervisor/guru/VMwareGuru.java | 3 + .../vmware/manager/VmwareManager.java | 1 + .../vmware/manager/VmwareManagerImpl.java | 51 +++++- .../manager/VmwareStorageManagerImpl.java | 87 ++++------ .../vmware/resource/VmwareContextFactory.java | 6 + .../vmware/resource/VmwareResource.java | 150 ++++++------------ ...VmwareSecondaryStorageResourceHandler.java | 1 + .../resource/VmwareStorageProcessor.java | 4 +- .../src/com/cloud/configuration/Config.java | 1 + .../vmware/mo/CustomFieldConstants.java | 2 + .../vmware/mo/HypervisorHostHelper.java | 20 ++- .../vmware/mo/VirtualMachineMO.java | 8 +- 16 files changed, 191 insertions(+), 176 deletions(-) diff --git a/agent/src/com/cloud/agent/resource/consoleproxy/ConsoleProxyResource.java b/agent/src/com/cloud/agent/resource/consoleproxy/ConsoleProxyResource.java index 991764c53f8..ee5c36176c8 100644 --- a/agent/src/com/cloud/agent/resource/consoleproxy/ConsoleProxyResource.java +++ b/agent/src/com/cloud/agent/resource/consoleproxy/ConsoleProxyResource.java @@ -112,6 +112,7 @@ public class ConsoleProxyResource extends ServerResourceBase implements } private Answer execute(StartConsoleProxyAgentHttpHandlerCommand cmd) { + s_logger.info("Invoke launchConsoleProxy() in responding to StartConsoleProxyAgentHttpHandlerCommand"); launchConsoleProxy(cmd.getKeystoreBits(), cmd.getKeystorePassword(), cmd.getEncryptorPassword()); return new Answer(cmd); } @@ -361,29 +362,31 @@ public class ConsoleProxyResource extends ServerResourceBase implements try { Class consoleProxyClazz = Class.forName("com.cloud.consoleproxy.ConsoleProxy"); try { + s_logger.info("Invoke setEncryptorPassword(), ecnryptorPassword: " + encryptorPassword); Method methodSetup = consoleProxyClazz.getMethod( "setEncryptorPassword", String.class); methodSetup.invoke(null, encryptorPassword); + s_logger.info("Invoke startWithContext()"); Method method = consoleProxyClazz.getMethod( "startWithContext", Properties.class, Object.class, byte[].class, String.class); method.invoke(null, _properties, resource, ksBits, ksPassword); } catch (SecurityException e) { - s_logger.error("Unable to launch console proxy due to SecurityException"); + s_logger.error("Unable to launch console proxy due to SecurityException", e); System.exit(ExitStatus.Error.value()); } catch (NoSuchMethodException e) { - s_logger.error("Unable to launch console proxy due to NoSuchMethodException"); + s_logger.error("Unable to launch console proxy due to NoSuchMethodException", e); System.exit(ExitStatus.Error.value()); } catch (IllegalArgumentException e) { - s_logger.error("Unable to launch console proxy due to IllegalArgumentException"); + s_logger.error("Unable to launch console proxy due to IllegalArgumentException", e); System.exit(ExitStatus.Error.value()); } catch (IllegalAccessException e) { - s_logger.error("Unable to launch console proxy due to IllegalAccessException"); + s_logger.error("Unable to launch console proxy due to IllegalAccessException", e); System.exit(ExitStatus.Error.value()); } catch (InvocationTargetException e) { - s_logger.error("Unable to launch console proxy due to InvocationTargetException"); + s_logger.error("Unable to launch console proxy due to InvocationTargetException " + e.getTargetException().toString(), e); System.exit(ExitStatus.Error.value()); } } catch (final ClassNotFoundException e) { @@ -402,22 +405,22 @@ public class ConsoleProxyResource extends ServerResourceBase implements Method methodSetup = consoleProxyClazz.getMethod("setEncryptorPassword", String.class); methodSetup.invoke(null, encryptorPassword); } catch (SecurityException e) { - s_logger.error("Unable to launch console proxy due to SecurityException"); + s_logger.error("Unable to launch console proxy due to SecurityException", e); System.exit(ExitStatus.Error.value()); } catch (NoSuchMethodException e) { - s_logger.error("Unable to launch console proxy due to NoSuchMethodException"); + s_logger.error("Unable to launch console proxy due to NoSuchMethodException", e); System.exit(ExitStatus.Error.value()); } catch (IllegalArgumentException e) { - s_logger.error("Unable to launch console proxy due to IllegalArgumentException"); + s_logger.error("Unable to launch console proxy due to IllegalArgumentException", e); System.exit(ExitStatus.Error.value()); } catch (IllegalAccessException e) { - s_logger.error("Unable to launch console proxy due to IllegalAccessException"); + s_logger.error("Unable to launch console proxy due to IllegalAccessException", e); System.exit(ExitStatus.Error.value()); } catch (InvocationTargetException e) { - s_logger.error("Unable to launch console proxy due to InvocationTargetException"); + s_logger.error("Unable to launch console proxy due to InvocationTargetException " + e.getTargetException().toString(), e); System.exit(ExitStatus.Error.value()); } catch (final ClassNotFoundException e) { - s_logger.error("Unable to launch console proxy due to ClassNotFoundException"); + s_logger.error("Unable to launch console proxy due to ClassNotFoundException", e); System.exit(ExitStatus.Error.value()); } } diff --git a/client/pom.xml b/client/pom.xml index 1afe5b898d3..119c96eddef 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -17,7 +17,7 @@ org.apache.cloudstack cloudstack 4.3.0-SNAPSHOT - + org.apache.cloudstack diff --git a/framework/cluster/src/com/cloud/cluster/ClusterManager.java b/framework/cluster/src/com/cloud/cluster/ClusterManager.java index d843f2a4129..0ef86ea4fd3 100644 --- a/framework/cluster/src/com/cloud/cluster/ClusterManager.java +++ b/framework/cluster/src/com/cloud/cluster/ClusterManager.java @@ -55,6 +55,8 @@ public interface ClusterManager extends Manager { ManagementServerHost getPeer(String peerName); String getSelfPeerName(); + long getManagementNodeId(); + long getCurrentRunId(); public interface Dispatcher { String getName(); diff --git a/framework/cluster/src/com/cloud/cluster/ClusterManagerImpl.java b/framework/cluster/src/com/cloud/cluster/ClusterManagerImpl.java index 98d5b79f1e7..0f1cf9da487 100644 --- a/framework/cluster/src/com/cloud/cluster/ClusterManagerImpl.java +++ b/framework/cluster/src/com/cloud/cluster/ClusterManagerImpl.java @@ -1076,6 +1076,10 @@ public class ClusterManagerImpl extends ManagerBase implements ClusterManager, C } return true; } + + public long getManagementNodeId() { + return _msId; + } public long getCurrentRunId() { return _runId; diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/guru/VMwareGuru.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/guru/VMwareGuru.java index 53207d78dcf..76ee59bc8b4 100644 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/guru/VMwareGuru.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/guru/VMwareGuru.java @@ -54,6 +54,7 @@ import com.cloud.agent.api.to.DataStoreTO; import com.cloud.agent.api.to.DataTO; import com.cloud.agent.api.to.NicTO; import com.cloud.agent.api.to.VirtualMachineTO; +import com.cloud.cluster.ClusterManager; import com.cloud.configuration.Config; import com.cloud.exception.InsufficientAddressCapacityException; import com.cloud.host.Host; @@ -115,6 +116,7 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru { PhysicalNetworkDao _physicalNetworkDao; @Inject PhysicalNetworkTrafficTypeDao _physicalNetworkTrafficTypeDao; + @Inject ClusterManager _clusterMgr; protected VMwareGuru() { super(); @@ -365,6 +367,7 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru { CommandExecLogVO execLog = new CommandExecLogVO(cmdTarget.first().getId(), cmdTarget.second().getId(), cmd.getClass().getSimpleName(), 1); _cmdExecLogDao.persist(execLog); cmd.setContextParam("execid", String.valueOf(execLog.getId())); + cmd.setContextParam("noderuninfo", String.format("%d-%d", _clusterMgr.getManagementNodeId(), _clusterMgr.getCurrentRunId())); if(cmd instanceof BackupSnapshotCommand || cmd instanceof CreatePrivateTemplateFromVolumeCommand || diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareManager.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareManager.java index f9f5f7e7e39..6c675990bb3 100755 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareManager.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareManager.java @@ -52,6 +52,7 @@ public interface VmwareManager { VmwareStorageManager getStorageManager(); void gcLeftOverVMs(VmwareContext context); + boolean needRecycle(String workerTag); Pair getAddiionalVncPortRange(); diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java index 753440e65bc..2649452611f 100755 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java @@ -59,6 +59,9 @@ import com.cloud.agent.api.Answer; import com.cloud.agent.api.Command; import com.cloud.agent.api.StartupCommand; import com.cloud.agent.api.StartupRoutingCommand; +import com.cloud.cluster.ClusterManager; +import com.cloud.cluster.ManagementServerHost; +import com.cloud.cluster.dao.ManagementServerHostPeerDao; import com.cloud.configuration.Config; import com.cloud.dc.ClusterDetailsDao; import com.cloud.dc.ClusterVO; @@ -153,6 +156,8 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw @Inject VmwareDatacenterDao _vmwareDcDao; @Inject VmwareDatacenterZoneMapDao _vmwareDcZoneMapDao; @Inject LegacyZoneDao _legacyZoneDao; + @Inject ManagementServerHostPeerDao _mshostPeerDao; + @Inject ClusterManager _clusterMgr; String _mountParent; StorageLayer _storage; @@ -166,6 +171,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw String _managemetPortGroupName; String _defaultSystemVmNicAdapterType = VirtualEthernetCardType.E1000.toString(); String _recycleHungWorker = "false"; + long _hungWorkerTimeout = 7200000; // 2 hour int _additionalPortRangeStart; int _additionalPortRangeSize; int _routerExtraPublicNics = 2; @@ -284,6 +290,10 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw if(_recycleHungWorker == null || _recycleHungWorker.isEmpty()) { _recycleHungWorker = "false"; } + + value = _configDao.getValue(Config.VmwareHungWorkerTimeout.key()); + if(value != null) + _hungWorkerTimeout = Long.parseLong(value) * 1000; _rootDiskController = _configDao.getValue(Config.VmwareRootDiskControllerType.key()); if(_rootDiskController == null || _rootDiskController.isEmpty()) { @@ -529,11 +539,50 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw return _storageMgr; } - @Override public void gcLeftOverVMs(VmwareContext context) { VmwareCleanupMaid.gcLeftOverVMs(context); } + + @Override + public boolean needRecycle(String workerTag) { + if(s_logger.isInfoEnabled()) + s_logger.info("Check to see if a worker VM with tag " + workerTag + " needs to be recycled"); + + if(workerTag == null || workerTag.isEmpty()) { + s_logger.error("Invalid worker VM tag " + workerTag); + return false; + } + + String tokens[] = workerTag.split("-"); + if(tokens.length != 3) { + s_logger.error("Invalid worker VM tag " + workerTag); + return false; + } + + long startTick = Long.parseLong(tokens[0]); + long msid = Long.parseLong(tokens[1]); + long runid = Long.parseLong(tokens[2]); + + if(_mshostPeerDao.countStateSeenInPeers(msid, runid, ManagementServerHost.State.Down) > 0) { + if(s_logger.isInfoEnabled()) + s_logger.info("Worker VM's owner management server node has been detected down from peer nodes, recycle it"); + return true; + } + + if(msid == _clusterMgr.getManagementNodeId() && runid != _clusterMgr.getCurrentRunId()) { + if(s_logger.isInfoEnabled()) + s_logger.info("Worker VM's owner management server has changed runid, recycle it"); + return true; + } + + if(System.currentTimeMillis() - startTick > _hungWorkerTimeout) { + if(s_logger.isInfoEnabled()) + s_logger.info("Worker VM expired, seconds elapsed: " + (System.currentTimeMillis() - startTick) / 1000); + return true; + } + return false; + } @Override public void prepareSecondaryStorageStore(String storageUrl) { diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareStorageManagerImpl.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareStorageManagerImpl.java index 01a126fa680..e11e76612e3 100644 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareStorageManagerImpl.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareStorageManagerImpl.java @@ -315,46 +315,41 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { String prevBackupUuid = cmd.getPrevBackupUuid(); VirtualMachineMO workerVm=null; String workerVMName = null; - String volumePath = cmd.getVolumePath(); - ManagedObjectReference morDs = null; - DatastoreMO dsMo=null; + String volumePath = cmd.getVolumePath(); + ManagedObjectReference morDs = null; + DatastoreMO dsMo=null; - // By default assume failure - String details = null; - boolean success = false; - String snapshotBackupUuid = null; + // By default assume failure + String details = null; + boolean success = false; + String snapshotBackupUuid = null; - VmwareContext context = hostService.getServiceContext(cmd); - VirtualMachineMO vmMo = null; - try { - VmwareHypervisorHost hyperHost = hostService.getHyperHost(context, cmd); - morDs = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, cmd.getPool().getUuid()); + VmwareContext context = hostService.getServiceContext(cmd); + VirtualMachineMO vmMo = null; + try { + VmwareHypervisorHost hyperHost = hostService.getHyperHost(context, cmd); + morDs = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, cmd.getPool().getUuid()); - try { - vmMo = hyperHost.findVmOnHyperHost(cmd.getVmName()); - if (vmMo == null) { - if(s_logger.isDebugEnabled()) - s_logger.debug("Unable to find owner VM for BackupSnapshotCommand on host " + hyperHost.getHyperHostName() + ", will try within datacenter"); + try { + vmMo = hyperHost.findVmOnHyperHost(cmd.getVmName()); + if (vmMo == null) { + if(s_logger.isDebugEnabled()) { + s_logger.debug("Unable to find owner VM for BackupSnapshotCommand on host " + hyperHost.getHyperHostName() + ", will try within datacenter"); + } - vmMo = hyperHost.findVmOnPeerHyperHost(cmd.getVmName()); - if(vmMo == null) { - dsMo = new DatastoreMO(hyperHost.getContext(), morDs); + vmMo = hyperHost.findVmOnPeerHyperHost(cmd.getVmName()); + if(vmMo == null) { + dsMo = new DatastoreMO(hyperHost.getContext(), morDs); - workerVMName = hostService.getWorkerName(context, cmd, 0); + workerVMName = hostService.getWorkerName(context, cmd, 0); + vmMo = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, workerVMName); + + if (vmMo == null) { + throw new Exception("Failed to find the newly create or relocated VM. vmName: " + workerVMName); + } + workerVm = vmMo; - // attach a volume to dummay wrapper VM for taking snapshot and exporting the VM for backup - if (!hyperHost.createBlankVm(workerVMName, null, 1, 512, 0, false, 4, 0, VirtualMachineGuestOsIdentifier.OTHER_GUEST.value(), morDs, false)) { - String msg = "Unable to create worker VM to execute BackupSnapshotCommand"; - s_logger.error(msg); - throw new Exception(msg); - } - vmMo = hyperHost.findVmOnHyperHost(workerVMName); - if (vmMo == null) { - throw new Exception("Failed to find the newly create or relocated VM. vmName: " + workerVMName); - } - workerVm = vmMo; - - // attach volume to worker VM + // attach volume to worker VM String datastoreVolumePath = getVolumePathInDatastore(dsMo, volumePath + ".vmdk"); vmMo.attachDisk(new String[] { datastoreVolumePath }, morDs); } @@ -1071,28 +1066,8 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { if (vmMo == null) { // create a dummy worker vm for attaching the volume DatastoreMO dsMo = new DatastoreMO(hyperHost.getContext(), morDs); - //restrict VM name to 32 chars, (else snapshot descriptor file name will be truncated to 32 chars of vm name) - VirtualMachineConfigSpec vmConfig = new VirtualMachineConfigSpec(); - vmConfig.setName(workerVmName); - vmConfig.setMemoryMB((long) 4); - vmConfig.setNumCPUs(1); - vmConfig.setGuestId(VirtualMachineGuestOsIdentifier.OTHER_GUEST.value()); - VirtualMachineFileInfo fileInfo = new VirtualMachineFileInfo(); - fileInfo.setVmPathName(String.format("[%s]", dsMo.getName())); - vmConfig.setFiles(fileInfo); - - // Scsi controller - VirtualLsiLogicController scsiController = new VirtualLsiLogicController(); - scsiController.setSharedBus(VirtualSCSISharing.NO_SHARING); - scsiController.setBusNumber(0); - scsiController.setKey(1); - VirtualDeviceConfigSpec scsiControllerSpec = new VirtualDeviceConfigSpec(); - scsiControllerSpec.setDevice(scsiController); - scsiControllerSpec.setOperation(VirtualDeviceConfigSpecOperation.ADD); - vmConfig.getDeviceChange().add(scsiControllerSpec); - - hyperHost.createVm(vmConfig); - workerVm = hyperHost.findVmOnHyperHost(workerVmName); + workerVm = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, workerVmName); + if (workerVm == null) { String msg = "Unable to create worker VM to execute CopyVolumeCommand"; s_logger.error(msg); diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareContextFactory.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareContextFactory.java index c0b716cc394..ed607e118d2 100755 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareContextFactory.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareContextFactory.java @@ -22,6 +22,7 @@ import javax.inject.Inject; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; +import com.cloud.cluster.ClusterManager; import com.cloud.hypervisor.vmware.manager.VmwareManager; import com.cloud.hypervisor.vmware.util.VmwareClient; import com.cloud.hypervisor.vmware.util.VmwareContext; @@ -34,9 +35,11 @@ public class VmwareContextFactory { private static volatile int s_seq = 1; private static VmwareManager s_vmwareMgr; + private static ClusterManager s_clusterMgr; private static VmwareContextPool s_pool; @Inject VmwareManager _vmwareMgr; + @Inject ClusterManager _clusterMgr; static { // skip certificate check @@ -47,6 +50,7 @@ public class VmwareContextFactory { @PostConstruct void init() { s_vmwareMgr = _vmwareMgr; + s_clusterMgr = _clusterMgr; } public static VmwareContext create(String vCenterAddress, String vCenterUserName, String vCenterPassword) throws Exception { @@ -66,6 +70,7 @@ public class VmwareContextFactory { context.registerStockObject("serviceconsole", s_vmwareMgr.getServiceConsolePortGroupName()); context.registerStockObject("manageportgroup", s_vmwareMgr.getManagementPortGroupName()); + context.registerStockObject("noderuninfo", String.format("%d-%d", s_clusterMgr.getManagementNodeId(), s_clusterMgr.getCurrentRunId())); context.setPoolInfo(s_pool, VmwareContextPool.composePoolKey(vCenterAddress, vCenterUserName)); s_pool.registerOutstandingContext(context); @@ -83,6 +88,7 @@ public class VmwareContextFactory { context.registerStockObject("serviceconsole", s_vmwareMgr.getServiceConsolePortGroupName()); context.registerStockObject("manageportgroup", s_vmwareMgr.getManagementPortGroupName()); + context.registerStockObject("noderuninfo", String.format("%d-%d", s_clusterMgr.getManagementNodeId(), s_clusterMgr.getCurrentRunId())); } return context; diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 450c5f09317..2253586fe56 100755 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -29,7 +29,6 @@ import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.Date; -import java.util.GregorianCalendar; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -284,7 +283,6 @@ import com.cloud.hypervisor.vmware.mo.DatastoreMO; import com.cloud.hypervisor.vmware.mo.DiskControllerType; import com.cloud.hypervisor.vmware.mo.FeatureKeyConstants; import com.cloud.hypervisor.vmware.mo.HostDatastoreSystemMO; -import com.cloud.hypervisor.vmware.mo.HostFirewallSystemMO; import com.cloud.hypervisor.vmware.mo.HostMO; import com.cloud.hypervisor.vmware.mo.HostStorageSystemMO; import com.cloud.hypervisor.vmware.mo.HypervisorHostHelper; @@ -322,7 +320,6 @@ import com.cloud.utils.DateUtil; import com.cloud.utils.NumbersUtil; import com.cloud.utils.Pair; import com.cloud.utils.StringUtils; -import com.cloud.utils.component.ComponentContext; import com.cloud.utils.db.DB; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.exception.ExceptionUtil; @@ -4462,7 +4459,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa if (!dsMo.fileExists(volumeDatastorePath)) { String dummyVmName = getWorkerName(context, cmd, 0); - VirtualMachineMO vmMo = prepareVolumeHostDummyVm(hyperHost, dsMo, dummyVmName); + VirtualMachineMO vmMo = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, dummyVmName); if (vmMo == null) { throw new Exception("Unable to create a dummy VM for volume creation"); @@ -5621,7 +5618,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa VirtualMachineMO vmMo = null; try { - vmMo = prepareVolumeHostDummyVm(hyperHost, dsMo, dummyVmName); + vmMo = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, dummyVmName); if (vmMo == null) { throw new Exception("Unable to create a dummy VM for volume creation"); } @@ -5678,7 +5675,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa String volumeDatastorePath = String.format("[%s] %s.vmdk", dsMo.getName(), volumeUuid); String dummyVmName = getWorkerName(context, cmd, 0); try { - vmMo = prepareVolumeHostDummyVm(hyperHost, dsMo, dummyVmName); + vmMo = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, dummyVmName); if (vmMo == null) { throw new Exception("Unable to create a dummy VM for volume creation"); } @@ -5713,34 +5710,6 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa return (int)(bytes / (1024L * 1024L)); } - protected VirtualMachineMO prepareVolumeHostDummyVm(VmwareHypervisorHost hyperHost, DatastoreMO dsMo, String vmName) throws Exception { - assert (hyperHost != null); - - VirtualMachineMO vmMo = null; - VirtualMachineConfigSpec vmConfig = new VirtualMachineConfigSpec(); - vmConfig.setName(vmName); - vmConfig.setMemoryMB((long) 4); // vmware request minimum of 4 MB - vmConfig.setNumCPUs(1); - vmConfig.setGuestId(VirtualMachineGuestOsIdentifier.OTHER_GUEST.value()); - VirtualMachineFileInfo fileInfo = new VirtualMachineFileInfo(); - fileInfo.setVmPathName(String.format("[%s]", dsMo.getName())); - vmConfig.setFiles(fileInfo); - - // Scsi controller - VirtualLsiLogicController scsiController = new VirtualLsiLogicController(); - scsiController.setSharedBus(VirtualSCSISharing.NO_SHARING); - scsiController.setBusNumber(0); - scsiController.setKey(1); - VirtualDeviceConfigSpec scsiControllerSpec = new VirtualDeviceConfigSpec(); - scsiControllerSpec.setDevice(scsiController); - scsiControllerSpec.setOperation(VirtualDeviceConfigSpecOperation.ADD); - - vmConfig.getDeviceChange().add(scsiControllerSpec ); - hyperHost.createVm(vmConfig); - vmMo = hyperHost.findVmOnHyperHost(vmName); - return vmMo; - } - @Override public void disconnected() { } @@ -5767,70 +5736,53 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa if(hyperHost.isHyperHostConnected()) { mgr.gcLeftOverVMs(context); - if(_recycleHungWorker) { - s_logger.info("Scan hung worker VM to recycle"); - - int key = ((HostMO)hyperHost).getCustomFieldKey("VirtualMachine", CustomFieldConstants.CLOUD_VM_INTERNAL_NAME); - if(key == 0) { - s_logger.warn("Custom field " + CustomFieldConstants.CLOUD_VM_INTERNAL_NAME + " is not registered ?!"); - } - String instanceNameCustomField = "value[" + key + "]"; - - // GC worker that has been running for too long - ObjectContent[] ocs = hyperHost.getVmPropertiesOnHyperHost( - new String[] {"name", "config.template", "runtime.powerState", "runtime.bootTime", instanceNameCustomField }); - if(ocs != null) { - for(ObjectContent oc : ocs) { - List props = oc.getPropSet(); - if(props != null) { - String vmName = null; - String internalName = null; - boolean template = false; - VirtualMachinePowerState powerState = VirtualMachinePowerState.POWERED_OFF; - GregorianCalendar bootTime = null; - - for(DynamicProperty prop : props) { - if (prop.getName().equals("name")) - vmName = prop.getVal().toString(); - else if(prop.getName().startsWith("value[")) { - if(prop.getVal() != null) - internalName = ((CustomFieldStringValue)prop.getVal()).getValue(); - } - else if(prop.getName().equals("config.template")) - template = (Boolean)prop.getVal(); - else if(prop.getName().equals("runtime.powerState")) - powerState = (VirtualMachinePowerState)prop.getVal(); - else if(prop.getName().equals("runtime.bootTime")) - bootTime = (GregorianCalendar)prop.getVal(); - } - - VirtualMachineMO vmMo = new VirtualMachineMO(hyperHost.getContext(), oc.getObj()); - String name = null; - if (internalName != null) { - name = internalName; - } else { - name = vmName; - } - - if(!template && name.matches("[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}")) { - boolean recycle = false; - - // recycle stopped worker VM and VM that has been running for too long (hard-coded 10 hours for now) - if(powerState == VirtualMachinePowerState.POWERED_OFF) - recycle = true; - else if(bootTime != null && (new Date().getTime() - bootTime.getTimeInMillis() > 10*3600*1000)) - recycle = true; - - if(recycle) { - s_logger.info("Recycle pending worker VM: " + name); - - vmMo.powerOff(); - vmMo.destroy(); - } - } - } - } - } + s_logger.info("Scan hung worker VM to recycle"); + + int workerKey = ((HostMO)hyperHost).getCustomFieldKey("VirtualMachine", CustomFieldConstants.CLOUD_WORKER); + int workerTagKey = ((HostMO)hyperHost).getCustomFieldKey("VirtualMachine", CustomFieldConstants.CLOUD_WORKER_TAG); + String workerPropName = String.format("value[%d]", workerKey); + String workerTagPropName = String.format("value[%d]", workerTagKey); + + // GC worker that has been running for too long + ObjectContent[] ocs = hyperHost.getVmPropertiesOnHyperHost( + new String[] {"name", "config.template", workerPropName, workerTagPropName, + }); + if(ocs != null) { + for(ObjectContent oc : ocs) { + List props = oc.getPropSet(); + if(props != null) { + boolean template = false; + boolean isWorker = false; + String workerTag = null; + + for(DynamicProperty prop : props) { + if(prop.getName().equals("config.template")) { + template = (Boolean)prop.getVal(); + } else if(prop.getName().equals(workerPropName)) { + CustomFieldStringValue val = (CustomFieldStringValue)prop.getVal(); + if(val != null && val.getValue() != null && val.getValue().equalsIgnoreCase("true")) + isWorker = true; + } + else if(prop.getName().equals(workerTagPropName)) { + CustomFieldStringValue val = (CustomFieldStringValue)prop.getVal(); + workerTag = val.getValue(); + } + } + + VirtualMachineMO vmMo = new VirtualMachineMO(hyperHost.getContext(), oc.getObj()); + if(!template && isWorker) { + boolean recycle = false; + recycle = mgr.needRecycle(workerTag); + + if(recycle) { + s_logger.info("Recycle pending worker VM: " + vmMo.getName()); + + vmMo.powerOff(); + vmMo.destroy(); + } + } + } + } } } else { s_logger.error("Host is no longer connected."); @@ -6613,6 +6565,8 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa cfmMo.ensureCustomFieldDef("VirtualMachine", CustomFieldConstants.CLOUD_UUID); cfmMo.ensureCustomFieldDef("VirtualMachine", CustomFieldConstants.CLOUD_NIC_MASK); cfmMo.ensureCustomFieldDef("VirtualMachine", CustomFieldConstants.CLOUD_VM_INTERNAL_NAME); + cfmMo.ensureCustomFieldDef("VirtualMachine", CustomFieldConstants.CLOUD_WORKER); + cfmMo.ensureCustomFieldDef("VirtualMachine", CustomFieldConstants.CLOUD_WORKER_TAG); VmwareHypervisorHost hostMo = this.getHyperHost(context); _hostName = hostMo.getHyperHostName(); diff --git a/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareSecondaryStorageResourceHandler.java b/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareSecondaryStorageResourceHandler.java index dcf71cb0e03..2c302ab29fc 100644 --- a/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareSecondaryStorageResourceHandler.java +++ b/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareSecondaryStorageResourceHandler.java @@ -218,6 +218,7 @@ public class VmwareSecondaryStorageResourceHandler implements SecondaryStorageRe if (context != null) { context.registerStockObject("serviceconsole", cmd.getContextParam("serviceconsole")); context.registerStockObject("manageportgroup", cmd.getContextParam("manageportgroup")); + context.registerStockObject("noderuninfo", cmd.getContextParam("noderuninfo")); } currentContext.set(context); return context; diff --git a/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageProcessor.java index d8ea1bdb40f..fd28e835ece 100644 --- a/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -178,7 +178,9 @@ public class VmwareStorageProcessor implements StorageProcessor { } if(vmMo.createSnapshot("cloud.template.base", "Base snapshot", false, false)) { - vmMo.setCustomFieldValue(CustomFieldConstants.CLOUD_UUID, templateUuid); + // the same template may be deployed with multiple copies at per-datastore per-host basis, + // save the original template name from CloudStack DB as the UUID to associate them. + vmMo.setCustomFieldValue(CustomFieldConstants.CLOUD_UUID, templateName); vmMo.markAsTemplate(); } else { vmMo.destroy(); diff --git a/server/src/com/cloud/configuration/Config.java b/server/src/com/cloud/configuration/Config.java index f867e2ce29e..8ca595b14d0 100755 --- a/server/src/com/cloud/configuration/Config.java +++ b/server/src/com/cloud/configuration/Config.java @@ -260,6 +260,7 @@ public enum Config { VmwareRootDiskControllerType("Advanced", ManagementServer.class, String.class, "vmware.root.disk.controller", "ide", "Specify the default disk controller for root volumes, valid values are scsi, ide", null), VmwareSystemVmNicDeviceType("Advanced", ManagementServer.class, String.class, "vmware.systemvm.nic.device.type", "E1000", "Specify the default network device type for system VMs, valid values are E1000, PCNet32, Vmxnet2, Vmxnet3", null), VmwareRecycleHungWorker("Advanced", ManagementServer.class, Boolean.class, "vmware.recycle.hung.wokervm", "false", "Specify whether or not to recycle hung worker VMs", null), + VmwareHungWorkerTimeout("Advanced", ManagementServer.class, Long.class, "vmware.hung.wokervm.timeout", "7200", "Worker VM timeout in seconds", null), VmwareEnableNestedVirtualization("Advanced", ManagementServer.class, Boolean.class, "vmware.nested.virtualization", "false", "When set to true this will enable nested virtualization when this is supported by the hypervisor", null), // Midonet diff --git a/vmware-base/src/com/cloud/hypervisor/vmware/mo/CustomFieldConstants.java b/vmware-base/src/com/cloud/hypervisor/vmware/mo/CustomFieldConstants.java index 139d377591c..47c2d3873ee 100644 --- a/vmware-base/src/com/cloud/hypervisor/vmware/mo/CustomFieldConstants.java +++ b/vmware-base/src/com/cloud/hypervisor/vmware/mo/CustomFieldConstants.java @@ -23,4 +23,6 @@ public interface CustomFieldConstants { public final static String CLOUD_NIC_MASK = "cloud.nic.mask"; public final static String CLOUD_ZONE = "cloud.zone"; public final static String CLOUD_VM_INTERNAL_NAME = "cloud.vm.internal.name"; + public final static String CLOUD_WORKER = "cloud.vm.worker"; + public final static String CLOUD_WORKER_TAG = "cloud.vm.worker.tag"; } diff --git a/vmware-base/src/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java b/vmware-base/src/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java index ac34b490be9..3a70744aa95 100755 --- a/vmware-base/src/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java +++ b/vmware-base/src/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java @@ -1233,8 +1233,24 @@ public class HypervisorHostHelper { scsiControllerSpec.setOperation(VirtualDeviceConfigSpecOperation.ADD); vmConfig.getDeviceChange().add(scsiControllerSpec); - hyperHost.createVm(vmConfig); - workingVM = hyperHost.findVmOnHyperHost(vmName); + if(hyperHost.createVm(vmConfig)) { + // Ugly work-around, it takes time for newly created VM to appear + for(int i = 0; i < 10 && workingVM == null; i++) { + workingVM = hyperHost.findVmOnHyperHost(vmName); + + try { + Thread.sleep(1000); + } catch(InterruptedException e) { + } + } + } + + if(workingVM != null) { + workingVM.setCustomFieldValue(CustomFieldConstants.CLOUD_WORKER, "true"); + String workerTag = String.format("%d-%s", System.currentTimeMillis(), + hyperHost.getContext().getStockObject("noderuninfo")); + workingVM.setCustomFieldValue(CustomFieldConstants.CLOUD_WORKER_TAG, workerTag); + } return workingVM; } diff --git a/vmware-base/src/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java b/vmware-base/src/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java index fcf2e5fb0ca..b9af73853d4 100644 --- a/vmware-base/src/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java +++ b/vmware-base/src/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java @@ -1559,15 +1559,11 @@ public class VirtualMachineMO extends BaseMO { assert(disks.length >= 1); HostMO hostMo = getRunningHost(); - VirtualMachineConfigInfo vmConfigInfo = getConfigInfo(); - if(!hostMo.createBlankVm(clonedVmName, null, 1, cpuSpeedMHz, 0, false, memoryMb, 0, vmConfigInfo.getGuestId(), morDs, false)) - throw new Exception("Unable to create a blank VM"); - - VirtualMachineMO clonedVmMo = hostMo.findVmOnHyperHost(clonedVmName); + VirtualMachineMO clonedVmMo = HypervisorHostHelper.createWorkerVM(hostMo, new DatastoreMO(hostMo.getContext(), morDs), clonedVmName); if(clonedVmMo == null) throw new Exception("Unable to find just-created blank VM"); - + boolean bSuccess = false; try { VirtualMachineConfigSpec vmConfigSpec = new VirtualMachineConfigSpec();