[Vmware] Improve listing of Vmware Datacenter VMs for migration to KVM (#10770)

Co-authored-by: dahn <daan.hoogland@gmail.com>
Co-authored-by: Suresh Kumar Anaparti <sureshkumar.anaparti@gmail.com>

api/src/main/java/org/apache/cloudstack/api/response/UnmanagedInstanceResponse.java

@@ -79,6 +79,14 @@ public class UnmanagedInstanceResponse extends BaseResponse {
     @Param(description = "the operating system of the virtual machine")
     private String operatingSystem;
 
+    @SerializedName(ApiConstants.BOOT_MODE)
+    @Param(description = "indicates the boot mode")
+    private String bootMode;
+
+    @SerializedName(ApiConstants.BOOT_TYPE)
+    @Param(description = "indicates the boot type")
+    private String bootType;
+
     @SerializedName(ApiConstants.DISK)
     @Param(description = "the list of disks associated with the virtual machine", responseObject = UnmanagedInstanceDiskResponse.class)
     private Set<UnmanagedInstanceDiskResponse> disks;
@@ -211,4 +219,20 @@ public class UnmanagedInstanceResponse extends BaseResponse {
     public void addNic(NicResponse nic) {
         this.nics.add(nic);
     }
+
+    public String getBootMode() {
+        return bootMode;
+    }
+
+    public void setBootMode(String bootMode) {
+        this.bootMode = bootMode;
+    }
+
+    public String getBootType() {
+        return bootType;
+    }
+
+    public void setBootType(String bootType) {
+        this.bootType = bootType;
+    }
 }

api/src/main/java/org/apache/cloudstack/vm/UnmanagedInstanceTO.java

@@ -61,6 +61,9 @@ public class UnmanagedInstanceTO {
 
     private String vncPassword;
 
+    private String bootType;
+    private String bootMode;
+
     public String getName() {
         return name;
     }
@@ -189,6 +192,22 @@ public class UnmanagedInstanceTO {
         this.vncPassword = vncPassword;
     }
 
+    public String getBootType() {
+        return bootType;
+    }
+
+    public void setBootType(String bootType) {
+        this.bootType = bootType;
+    }
+
+    public String getBootMode() {
+        return bootMode;
+    }
+
+    public void setBootMode(String bootMode) {
+        this.bootMode = bootMode;
+    }
+
     public static class Disk {
         private String diskId;
 

plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java

@@ -43,6 +43,7 @@ import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 import javax.persistence.EntityExistsException;
 
+import com.cloud.hypervisor.vmware.mo.VirtualMachineMO;
 import com.cloud.hypervisor.vmware.util.VmwareClient;
 import org.apache.cloudstack.api.command.admin.zone.AddVmwareDcCmd;
 import org.apache.cloudstack.api.command.admin.zone.ImportVsphereStoragePoliciesCmd;
@@ -1587,14 +1588,26 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw
         return compatiblePools;
     }
 
-    @Override
+    private static class VcenterData {
-    public List<UnmanagedInstanceTO> listVMsInDatacenter(ListVmwareDcVmsCmd cmd) {
+        public final String vcenter;
+        public final String datacenterName;
+        public final String username;
+        public final String password;
+
+        public VcenterData(String vcenter, String datacenterName, String username, String password) {
+            this.vcenter = vcenter;
+            this.datacenterName = datacenterName;
+            this.username = username;
+            this.password = password;
+        }
+    }
+
+    private VcenterData getVcenterData(ListVmwareDcVmsCmd cmd) {
         String vcenter = cmd.getVcenter();
         String datacenterName = cmd.getDatacenterName();
         String username = cmd.getUsername();
         String password = cmd.getPassword();
         Long existingVcenterId = cmd.getExistingVcenterId();
-        String keyword = cmd.getKeyword();
 
         if ((existingVcenterId == null && StringUtils.isBlank(vcenter)) ||
                 (existingVcenterId != null && StringUtils.isNotBlank(vcenter))) {
@@ -1615,34 +1628,67 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw
             username = vmwareDc.getUser();
             password = vmwareDc.getPassword();
         }
+        VcenterData vmwaredc = new VcenterData(vcenter, datacenterName, username, password);
+        return vmwaredc;
+    }
+
+    private static VmwareContext getVmwareContext(String vcenter, String username, String password) throws Exception {
+        s_logger.debug(String.format("Connecting to the VMware vCenter %s", vcenter));
+        String serviceUrl = String.format("https://%s/sdk/vimService", vcenter);
+        VmwareClient vimClient = new VmwareClient(vcenter);
+        vimClient.connect(serviceUrl, username, password);
+        return new VmwareContext(vimClient, vcenter);
+    }
+
+    @Override
+    public List<UnmanagedInstanceTO> listVMsInDatacenter(ListVmwareDcVmsCmd cmd) {
+        VcenterData vmwareDC = getVcenterData(cmd);
+        String vcenter = vmwareDC.vcenter;
+        String username = vmwareDC.username;
+        String password = vmwareDC.password;
+        String datacenterName = vmwareDC.datacenterName;
+        String keyword = cmd.getKeyword();
+        String esxiHostName = cmd.getHostName();
+        String virtualMachineName = cmd.getInstanceName();
 
         try {
-            s_logger.debug(String.format("Connecting to the VMware datacenter %s at vCenter %s to retrieve VMs",
+            VmwareContext context = getVmwareContext(vcenter, username, password);
-                    datacenterName, vcenter));
+            DatacenterMO dcMo = getDatacenterMO(context, vcenter, datacenterName);
-            String serviceUrl = String.format("https://%s/sdk/vimService", vcenter);
-            VmwareClient vimClient = new VmwareClient(vcenter);
-            vimClient.connect(serviceUrl, username, password);
-            VmwareContext context = new VmwareContext(vimClient, vcenter);
 
-            DatacenterMO dcMo = new DatacenterMO(context, datacenterName);
+            List<UnmanagedInstanceTO> instances;
-            ManagedObjectReference dcMor = dcMo.getMor();
+            if (StringUtils.isNotBlank(esxiHostName) && StringUtils.isNotBlank(virtualMachineName)) {
-            if (dcMor == null) {
+                ManagedObjectReference hostMor = dcMo.findHost(esxiHostName);
-                String msg = String.format("Unable to find VMware datacenter %s in vCenter %s",
+                if (hostMor == null) {
-                        datacenterName, vcenter);
+                    String errorMsg = String.format("Cannot find a host with name %s on vcenter %s", esxiHostName, vcenter);
-                s_logger.error(msg);
+                    s_logger.error(errorMsg);
-                throw new InvalidParameterValueException(msg);
+                    throw new CloudRuntimeException(errorMsg);
+                }
+                HostMO hostMO = new HostMO(context, hostMor);
+                VirtualMachineMO vmMo = hostMO.findVmOnHyperHost(virtualMachineName);
+                instances = Collections.singletonList(VmwareHelper.getUnmanagedInstance(hostMO, vmMo));
+            } else {
+                instances = dcMo.getAllVmsOnDatacenter(keyword);
             }
-            List<UnmanagedInstanceTO> instances = dcMo.getAllVmsOnDatacenter();
+            return instances;
-            return StringUtils.isBlank(keyword) ? instances :
-                    instances.stream().filter(x -> x.getName().toLowerCase().contains(keyword.toLowerCase())).collect(Collectors.toList());
         } catch (Exception e) {
-            String errorMsg = String.format("Error retrieving stopped VMs from the VMware VC %s datacenter %s: %s",
+            String errorMsg = String.format("Error retrieving VMs from the VMware VC %s datacenter %s: %s",
                     vcenter, datacenterName, e.getMessage());
             s_logger.error(errorMsg, e);
             throw new CloudRuntimeException(errorMsg);
         }
     }
 
+    private static DatacenterMO getDatacenterMO(VmwareContext context, String vcenter, String datacenterName) throws Exception {
+        DatacenterMO dcMo = new DatacenterMO(context, datacenterName);
+        ManagedObjectReference dcMor = dcMo.getMor();
+        if (dcMor == null) {
+            String msg = String.format("Unable to find VMware datacenter %s in vCenter %s", datacenterName, vcenter);
+            s_logger.error(msg);
+            throw new InvalidParameterValueException(msg);
+        }
+        return dcMo;
+    }
+
     @Override
     public boolean hasNexusVSM(Long clusterId) {
         ClusterVSMMapVO vsmMapVo = null;

plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVmwareDcVmsCmd.java

@@ -70,6 +70,12 @@ public class ListVmwareDcVmsCmd extends BaseListCmd {
     @Parameter(name = ApiConstants.PASSWORD, type = CommandType.STRING, description = "The password for specified username.")
     private String password;
 
+    @Parameter(name = ApiConstants.HOST_NAME, type = CommandType.STRING, description = "Name of the host on vCenter. Must be set along with the instancename parameter")
+    private String hostName;
+
+    @Parameter(name = ApiConstants.INSTANCE_NAME, type = CommandType.STRING, description = "Name of the VM on vCenter. Must be set along with the hostname parameter")
+    private String instanceName;
+
     public String getVcenter() {
         return vcenter;
     }
@@ -86,10 +92,18 @@ public class ListVmwareDcVmsCmd extends BaseListCmd {
         return datacenterName;
     }
 
+    public String getHostName() {
+        return hostName;
+    }
+
     public Long getExistingVcenterId() {
         return existingVcenterId;
     }
 
+    public String getInstanceName() {
+        return instanceName;
+    }
+
     @Override
     public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException {
         checkParameters();
@@ -125,6 +139,11 @@ public class ListVmwareDcVmsCmd extends BaseListCmd {
             throw new ServerApiException(ApiErrorCode.PARAM_ERROR,
                     "Please set all the information for a vCenter IP/Name, datacenter, username and password");
         }
+        if ((StringUtils.isNotBlank(instanceName) && StringUtils.isBlank(hostName)) ||
+                (StringUtils.isBlank(instanceName) && StringUtils.isNotBlank(hostName))) {
+            throw new ServerApiException(ApiErrorCode.PARAM_ERROR,
+                    "Please set the hostname parameter along with the instancename parameter");
+        }
     }
 
     @Override

server/src/main/java/com/cloud/api/ApiResponseHelper.java

@@ -5145,6 +5145,8 @@ public class ApiResponseHelper implements ResponseGenerator {
         response.setMemory(instance.getMemory());
         response.setOperatingSystemId(instance.getOperatingSystemId());
         response.setOperatingSystem(instance.getOperatingSystem());
+        response.setBootMode(instance.getBootMode());
+        response.setBootType(instance.getBootType());
         response.setObjectName("unmanagedinstance");
 
         if (instance.getDisks() != null) {

ui/src/views/tools/ManageInstances.vue

@@ -1302,6 +1302,31 @@ export default {
         this.fetchInstances()
       }
     },
+    fetchVmwareInstanceForKVMMigration (vmname, hostname) {
+      const params = {}
+      if (this.isMigrateFromVmware && this.selectedVmwareVcenter) {
+        if (this.selectedVmwareVcenter.vcenter) {
+          params.datacentername = this.selectedVmwareVcenter.datacentername
+          params.vcenter = this.selectedVmwareVcenter.vcenter
+          params.username = this.selectedVmwareVcenter.username
+          params.password = this.selectedVmwareVcenter.password
+        } else {
+          params.existingvcenterid = this.selectedVmwareVcenter.existingvcenterid
+        }
+        params.instancename = vmname
+        params.hostname = hostname
+      }
+      api('listVmwareDcVms', params).then(json => {
+        const response = json.listvmwaredcvmsresponse
+        this.selectedUnmanagedInstance = response.unmanagedinstance[0]
+        this.selectedUnmanagedInstance.ostypename = this.selectedUnmanagedInstance.osdisplayname
+        this.selectedUnmanagedInstance.state = this.selectedUnmanagedInstance.powerstate
+      }).catch(error => {
+        this.$notifyError(error)
+      }).finally(() => {
+        this.loading = false
+      })
+    },
     onManageInstanceAction () {
       this.selectedUnmanagedInstance = {}
       if (this.unmanagedInstances.length > 0 &&
@@ -1319,6 +1344,9 @@ export default {
           }
         })
         this.showUnmanageForm = false
+      } else if (this.isMigrateFromVmware) {
+        this.fetchVmwareInstanceForKVMMigration(this.selectedUnmanagedInstance.name, this.selectedUnmanagedInstance.hostname)
+        this.showUnmanageForm = true
       } else {
         this.showUnmanageForm = true
       }

ui/src/views/tools/SelectVmwareVcenter.vue

@@ -227,6 +227,8 @@ export default {
       } else {
         params.existingvcenterid = this.selectedExistingVcenterId
       }
+      params.page = 1
+      params.pagesize = 10
       api('listVmwareDcVms', params).then(json => {
         const obj = {
           params: params,
@@ -265,6 +267,11 @@ export default {
         this.loading = false
       })
     },
+    onSelectExternalVmwareDatacenter (value) {
+      if (this.vcenterSelectedOption === 'new' && !(this.vcenter === '' || this.datacentername === '' || this.username === '' || this.password === '')) {
+        this.listVmwareDatacenterVms()
+      }
+    },
     onSelectExistingVmwareDatacenter (value) {
       this.selectedExistingVcenterId = value
     },

vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/BaseMO.java

@@ -16,6 +16,13 @@
 // under the License.
 package com.cloud.hypervisor.vmware.mo;
 
+import com.cloud.utils.Pair;
+import com.vmware.vim25.DynamicProperty;
+import com.vmware.vim25.ObjectContent;
+import com.vmware.vim25.VirtualMachineBootOptions;
+import com.vmware.vim25.VirtualMachinePowerState;
+import org.apache.cloudstack.vm.UnmanagedInstanceTO;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.log4j.Logger;
 
 import com.vmware.vim25.CustomFieldDef;
@@ -24,12 +31,22 @@ import com.vmware.vim25.ManagedObjectReference;
 
 import com.cloud.hypervisor.vmware.util.VmwareContext;
 
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
 public class BaseMO {
     private static final Logger s_logger = Logger.getLogger(BaseMO.class);
 
     protected VmwareContext _context;
     protected ManagedObjectReference _mor;
 
+    protected static String[] propertyPathsForUnmanagedVmsThinListing = new String[] {"name", "config.template",
+            "runtime.powerState", "config.guestId", "config.guestFullName", "runtime.host",
+            "config.bootOptions", "config.firmware"};
+
     private String _name;
 
     public BaseMO(VmwareContext context, ManagedObjectReference mor) {
@@ -153,4 +170,94 @@ public class BaseMO {
 
         return cfmMo.getCustomFieldKey(morType, fieldName);
     }
+
+    private static UnmanagedInstanceTO.PowerState convertPowerState(VirtualMachinePowerState powerState) {
+        return powerState == VirtualMachinePowerState.POWERED_ON ? UnmanagedInstanceTO.PowerState.PowerOn :
+                powerState == VirtualMachinePowerState.POWERED_OFF ? UnmanagedInstanceTO.PowerState.PowerOff : UnmanagedInstanceTO.PowerState.PowerUnknown;
+    }
+
+    protected List<UnmanagedInstanceTO> convertVmsObjectContentsToUnmanagedInstances(List<ObjectContent> ocs, String keyword) throws Exception {
+        Map<String, Pair<String, String>> hostClusterNamesMap = new HashMap<>();
+        List<UnmanagedInstanceTO> vms = new ArrayList<>();
+        if (ocs != null) {
+            for (ObjectContent oc : ocs) {
+                List<DynamicProperty> objProps = oc.getPropSet();
+                if (objProps != null) {
+                    UnmanagedInstanceTO vm = createUnmanagedInstanceTOFromThinListingDynamicProperties(
+                            objProps, keyword, hostClusterNamesMap);
+                    if (vm != null) {
+                        vms.add(vm);
+                    }
+                }
+            }
+        }
+        if (vms.size() > 0) {
+            vms.sort(Comparator.comparing(UnmanagedInstanceTO::getName));
+        }
+        return vms;
+    }
+
+    private UnmanagedInstanceTO createUnmanagedInstanceTOFromThinListingDynamicProperties(List<DynamicProperty> objProps,
+                                                                                            String keyword,
+                                                                                            Map<String, Pair<String, String>> hostClusterNamesMap) throws Exception {
+        UnmanagedInstanceTO vm = new UnmanagedInstanceTO();
+        String vmName;
+        boolean isTemplate = false;
+        boolean excludeByKeyword = false;
+
+        for (DynamicProperty objProp : objProps) {
+            if (objProp.getName().equals("name")) {
+                vmName = (String) objProp.getVal();
+                if (StringUtils.isNotBlank(keyword) && !vmName.contains(keyword)) {
+                    excludeByKeyword = true;
+                }
+                vm.setName(vmName);
+            } else if (objProp.getName().equals("config.template")) {
+                isTemplate = (Boolean) objProp.getVal();
+            } else if (objProp.getName().equals("runtime.powerState")) {
+                VirtualMachinePowerState powerState = (VirtualMachinePowerState) objProp.getVal();
+                vm.setPowerState(convertPowerState(powerState));
+            } else if (objProp.getName().equals("config.guestFullName")) {
+                vm.setOperatingSystem((String) objProp.getVal());
+            } else if (objProp.getName().equals("config.guestId")) {
+                vm.setOperatingSystemId((String) objProp.getVal());
+            } else if (objProp.getName().equals("config.bootOptions")) {
+                VirtualMachineBootOptions bootOptions = (VirtualMachineBootOptions) objProp.getVal();
+                String bootMode = "LEGACY";
+                if (bootOptions != null && bootOptions.isEfiSecureBootEnabled()) {
+                    bootMode = "SECURE";
+                }
+                vm.setBootMode(bootMode);
+            } else if (objProp.getName().equals("config.firmware")) {
+                String firmware = (String) objProp.getVal();
+                vm.setBootType(firmware.equalsIgnoreCase("efi") ? "UEFI" : "BIOS");
+            } else if (objProp.getName().equals("runtime.host")) {
+                ManagedObjectReference hostMor = (ManagedObjectReference) objProp.getVal();
+                setUnmanagedInstanceTOHostAndCluster(vm, hostMor, hostClusterNamesMap);
+            }
+        }
+        if (isTemplate || excludeByKeyword) {
+            return null;
+        }
+        return vm;
+    }
+
+    private void setUnmanagedInstanceTOHostAndCluster(UnmanagedInstanceTO vm, ManagedObjectReference hostMor,
+                                                      Map<String, Pair<String, String>> hostClusterNamesMap) throws Exception {
+        if (hostMor != null && StringUtils.isNotBlank(hostMor.getValue())) {
+            String hostMorValue = hostMor.getValue();
+            Pair<String, String> hostClusterPair;
+            if (hostClusterNamesMap.containsKey(hostMorValue)) {
+                hostClusterPair = hostClusterNamesMap.get(hostMorValue);
+            } else {
+                HostMO hostMO = new HostMO(_context, hostMor);
+                ClusterMO clusterMO = new ClusterMO(_context, hostMO.getHyperHostCluster());
+                hostClusterPair = new Pair<>(hostMO.getHostName(), clusterMO.getName());
+                hostClusterNamesMap.put(hostMorValue, hostClusterPair);
+            }
+            vm.setHostName(hostClusterPair.first());
+            vm.setClusterName(hostClusterPair.second());
+        }
+    }
+
 }

vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatacenterMO.java

@@ -21,7 +21,6 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
-import com.cloud.hypervisor.vmware.util.VmwareHelper;
 import org.apache.cloudstack.vm.UnmanagedInstanceTO;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.log4j.Logger;
@@ -161,28 +160,9 @@ public class DatacenterMO extends BaseMO {
         return null;
     }
 
-    public List<UnmanagedInstanceTO> getAllVmsOnDatacenter() throws Exception {
+    public List<UnmanagedInstanceTO> getAllVmsOnDatacenter(String keyword) throws Exception {
-        List<UnmanagedInstanceTO> vms = new ArrayList<>();
+        List<ObjectContent> ocs = getVmPropertiesOnDatacenterVmFolder(propertyPathsForUnmanagedVmsThinListing);
-        List<ObjectContent> ocs = getVmPropertiesOnDatacenterVmFolder(new String[] {"name"});
+        return convertVmsObjectContentsToUnmanagedInstances(ocs, keyword);
-        if (ocs != null) {
-            for (ObjectContent oc : ocs) {
-                ManagedObjectReference vmMor = oc.getObj();
-                if (vmMor != null) {
-                    VirtualMachineMO vmMo = new VirtualMachineMO(_context, vmMor);
-                    try {
-                        if (!vmMo.isTemplate()) {
-                            HostMO hostMO = vmMo.getRunningHost();
-                            UnmanagedInstanceTO unmanagedInstance = VmwareHelper.getUnmanagedInstance(hostMO, vmMo);
-                            vms.add(unmanagedInstance);
-                        }
-                    } catch (Exception e) {
-                        s_logger.debug(String.format("Unexpected error checking unmanaged instance %s, excluding it: %s", vmMo.getVmName(), e.getMessage()), e);
-                    }
-                }
-            }
-        }
-
-        return vms;
     }
 
     public List<HostMO> getAllHostsOnDatacenter() throws Exception {

vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java

@@ -59,6 +59,8 @@ import com.vmware.vim25.NasDatastoreInfo;
 import com.vmware.vim25.VMwareDVSPortSetting;
 import com.vmware.vim25.VirtualDeviceFileBackingInfo;
 import com.vmware.vim25.VirtualIDEController;
+import com.vmware.vim25.VirtualMachineBootOptions;
+import com.vmware.vim25.VirtualMachineConfigInfo;
 import com.vmware.vim25.VirtualMachineConfigSummary;
 import com.vmware.vim25.VirtualMachineGuestOsIdentifier;
 import com.vmware.vim25.VirtualMachineToolsStatus;
@@ -810,6 +812,17 @@ public class VmwareHelper {
                 instance.setCpuSpeed(configSummary.getCpuReservation());
                 instance.setMemory(configSummary.getMemorySizeMB());
             }
+            VirtualMachineConfigInfo configInfo = vmMo.getConfigInfo();
+            if (configInfo != null) {
+                String firmware = configInfo.getFirmware();
+                instance.setBootType(firmware.equalsIgnoreCase("efi") ? "UEFI" : "BIOS");
+                VirtualMachineBootOptions bootOptions = configInfo.getBootOptions();
+                String bootMode = "LEGACY";
+                if (bootOptions != null && bootOptions.isEfiSecureBootEnabled()) {
+                    bootMode = "SECURE";
+                }
+                instance.setBootMode(bootMode);
+            }
 
             try {
                 ClusterMO clusterMo = new ClusterMO(hyperHost.getContext(), hyperHost.getHyperHostCluster());