From 0701dc9d9cfeaafbb957b46994826ef975c677e9 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Fri, 22 May 2020 11:01:46 +0530 Subject: [PATCH 001/164] Initial commit adding few required managed object classes and added pbm sdk --- deps/install-non-oss.sh | 2 + plugins/hypervisors/vmware/pom.xml | 6 ++ vmware-base/pom.xml | 6 ++ .../vmware/mo/PbmProfileManagerMO.java | 53 +++++++++++++++++ .../vmware/mo/VirtualMachineMO.java | 10 +++- .../mo/VirtualStorageObjectManager.java | 58 +++++++++++++++++++ .../hypervisor/vmware/util/VmwareClient.java | 26 +++++++++ .../hypervisor/vmware/util/VmwareContext.java | 10 ++++ 8 files changed, 170 insertions(+), 1 deletion(-) create mode 100644 vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/PbmProfileManagerMO.java create mode 100644 vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualStorageObjectManager.java diff --git a/deps/install-non-oss.sh b/deps/install-non-oss.sh index a387050d539..f8a526305a8 100755 --- a/deps/install-non-oss.sh +++ b/deps/install-non-oss.sh @@ -39,3 +39,5 @@ mvn install:install-file -Dfile=vim25_65.jar -DgroupId=com.cloud.com.vmwa # From https://my.vmware.com/group/vmware/details?downloadGroup=WEBCLIENTSDK67U2&productId=742 mvn install:install-file -Dfile=vim25_67.jar -DgroupId=com.cloud.com.vmware -DartifactId=vmware-vim25 -Dversion=6.7 -Dpackaging=jar + +mvn install:install-file -Dfile=pbm_65.jar -DgroupId=com.cloud.com.vmware -DartifactId=pbm -Dversion=6.5 -Dpackaging=jar diff --git a/plugins/hypervisors/vmware/pom.xml b/plugins/hypervisors/vmware/pom.xml index f5489488f35..b0c1ca8cb79 100644 --- a/plugins/hypervisors/vmware/pom.xml +++ b/plugins/hypervisors/vmware/pom.xml @@ -72,5 +72,11 @@ wsdl4j wsdl4j + + com.cloud.com.vmware + pbm + ${cs.vmware.api.version} + compile + diff --git a/vmware-base/pom.xml b/vmware-base/pom.xml index c84580f90b4..925a0031400 100644 --- a/vmware-base/pom.xml +++ b/vmware-base/pom.xml @@ -75,5 +75,11 @@ ${cs.jaxws.version} pom + + com.cloud.com.vmware + pbm + ${cs.vmware.api.version} + compile + diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/PbmProfileManagerMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/PbmProfileManagerMO.java new file mode 100644 index 00000000000..352cbf30027 --- /dev/null +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/PbmProfileManagerMO.java @@ -0,0 +1,53 @@ +package com.cloud.hypervisor.vmware.mo; + + +import com.cloud.hypervisor.vmware.util.VmwareContext; + +import com.vmware.pbm.PbmProfile; +import com.vmware.pbm.PbmProfileId; +import com.vmware.pbm.PbmProfileResourceType; +import com.vmware.pbm.PbmProfileResourceTypeEnum; +import com.vmware.vim25.ManagedObjectReference; + +import org.apache.log4j.Logger; + +import java.util.List; + +public class PbmProfileManagerMO extends BaseMO { + + private static final Logger s_logger = Logger.getLogger(PbmProfileManagerMO.class); + + public PbmProfileManagerMO (VmwareContext context) { + super(context, context.getPbmServiceContent().getProfileManager()); + } + + public PbmProfileManagerMO (VmwareContext context, ManagedObjectReference morProfileMgr) { + super(context, morProfileMgr); + } + + public PbmProfileManagerMO (VmwareContext context, String morType, String morValue) { + super(context, morType, morValue); + } + + public List getProfileIds() throws Exception { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Querying vCenter " + _context.getServerAddress() + " for profiles"); + } + List profileIds = _context.getPbmService().pbmQueryProfile(_mor, getStorageResourceType(), null); + return profileIds; + } + + public List getProfiles(PbmProfileResourceType pbmResourceType) throws Exception { + List profileIds = getProfileIds(); + List profiles = _context.getPbmService().pbmRetrieveContent(_mor, profileIds); + return profiles; + } + + private PbmProfileResourceType getStorageResourceType() { + PbmProfileResourceType resourceType = new PbmProfileResourceType(); + resourceType.setResourceType(PbmProfileResourceTypeEnum.STORAGE.value()); + return resourceType; + } +} + + diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java index b0507965c83..e9c8aafcbd1 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java @@ -34,6 +34,8 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import com.vmware.vim25.VStorageObject; +import com.vmware.vim25.VStorageObjectConfigInfo; import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; @@ -2485,7 +2487,13 @@ public class VirtualMachineMO extends BaseMO { String deviceNumbering = getDeviceBusName(devices, device); s_logger.info("Disk backing : " + diskBackingInfo.getFileName() + " matches ==> " + deviceNumbering); - + if (((VirtualDisk) device).getVDiskId() == null) { + s_logger.debug("vDiskid does not exist for volume " + vmdkDatastorePath + " registering the disk now"); + VirtualStorageObjectManager vStorageObjectManagerMO = new VirtualStorageObjectManager(getOwnerDatacenter().first().getContext()); + VStorageObject vStorageObject = vStorageObjectManagerMO.registerVirtualDisk(dsBackingFile, null, getOwnerDatacenter().first().getName()); + VStorageObjectConfigInfo diskConfigInfo = vStorageObject.getConfig(); + ((VirtualDisk) device).setVDiskId(diskConfigInfo.getId()); + } return new Pair<>((VirtualDisk)device, deviceNumbering); } diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualStorageObjectManager.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualStorageObjectManager.java new file mode 100644 index 00000000000..0b6f44ae0b0 --- /dev/null +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualStorageObjectManager.java @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.hypervisor.vmware.mo; + +import com.vmware.vim25.ID; +import com.vmware.vim25.VStorageObject; +import org.apache.log4j.Logger; + +import com.vmware.vim25.ManagedObjectReference; + +import com.cloud.hypervisor.vmware.util.VmwareContext; + +public class VirtualStorageObjectManager extends BaseMO { + @SuppressWarnings("unused") + private static final Logger s_logger = Logger.getLogger(VirtualStorageObjectManager.class); + + public VirtualStorageObjectManager(VmwareContext context) { + super(context, context.getServiceContent().getVStorageObjectManager()); + } + + public VirtualStorageObjectManager(VmwareContext context, ManagedObjectReference morDiskMgr) { + super(context, morDiskMgr); + } + + public VirtualStorageObjectManager(VmwareContext context, String morType, String morValue) { + super(context, morType, morValue); + } + + public VStorageObject registerVirtualDisk(DatastoreFile datastoreFile, String name, String dcName) throws Exception { + StringBuilder sb = new StringBuilder(); + //https://10.2.2.254/folder/i-2-4-VM/89e3756d9b7444dc92388eb36ddd026b.vmdk?dcPath=datacenter-21&dsName=c84e4af9b6ac33e887a25d9242650091 + sb.append("https://").append(_context.getServerAddress()).append("/folder/"); + sb.append(datastoreFile.getRelativePath()); + sb.append("?dcPath="); + sb.append(dcName); + sb.append("&dsName="); + sb.append(datastoreFile.getDatastoreName()); + return _context.getService().registerDisk(_mor, sb.toString(), name); + } + + public VStorageObject retrieveVirtualDisk (ID id, ManagedObjectReference morDS) throws Exception { + return _context.getService().retrieveVStorageObject(_mor, id, morDS); + } +} diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClient.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClient.java index 3d80ffdfae7..376afb1dbe7 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClient.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClient.java @@ -32,6 +32,8 @@ import javax.xml.ws.handler.MessageContext; import org.apache.cloudstack.utils.security.SSLUtils; import org.apache.cloudstack.utils.security.SecureSSLSocketFactory; +import com.vmware.pbm.PbmPortType; +import com.vmware.pbm.PbmServiceInstanceContent; import org.apache.log4j.Logger; import org.w3c.dom.Element; @@ -120,8 +122,14 @@ public class VmwareClient { } private final ManagedObjectReference svcInstRef = new ManagedObjectReference(); + private final ManagedObjectReference pbmSvcInstRef = new ManagedObjectReference(); + private static VimService vimService; private VimPortType vimPort; + private PbmPortType pbmPort; + private static final String PBM_SERVICE_INSTANCE_TYPE = "PbmServiceInstance"; + private static final String PBM_SERVICE_INSTANCE_VALUE = "ServiceInstance"; + private String serviceCookie; private final static String SVC_INST_NAME = "ServiceInstance"; private int vCenterSessionTimeout = 1200000; // Timeout in milliseconds @@ -211,6 +219,24 @@ public class VmwareClient { return null; } + /** + * @return PBM service instance + */ + public PbmPortType getPbmService() { + return pbmPort; + } + + /** + * @return Service instance content + */ + public PbmServiceInstanceContent getPbmServiceContent() { + try { + return pbmPort.pbmRetrieveServiceContent(pbmSvcInstRef); + } catch (com.vmware.pbm.RuntimeFaultFaultMsg e) { + } + return null; + } + /** * @return cookie used in service connection */ diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareContext.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareContext.java index 9b477aef42b..33b9644a307 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareContext.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareContext.java @@ -20,6 +20,8 @@ import com.cloud.hypervisor.vmware.mo.DatacenterMO; import com.cloud.hypervisor.vmware.mo.DatastoreFile; import com.cloud.utils.ActionDelegate; import com.cloud.utils.StringUtils; +import com.vmware.pbm.PbmPortType; +import com.vmware.pbm.PbmServiceInstanceContent; import com.vmware.vim25.ManagedObjectReference; import com.vmware.vim25.ObjectContent; import com.vmware.vim25.ObjectSpec; @@ -148,6 +150,14 @@ public class VmwareContext { return _vimClient.getServiceContent(); } + public PbmPortType getPbmService() { + return _vimClient.getPbmService(); + } + + public PbmServiceInstanceContent getPbmServiceContent() { + return _vimClient.getPbmServiceContent(); + } + public ManagedObjectReference getPropertyCollector() { return _vimClient.getPropCol(); } From f05b567d4c1335a16bfc8786eb9046a7815937f1 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Tue, 26 May 2020 19:09:40 +0530 Subject: [PATCH 002/164] Added PBM service connect --- .../vmware/mo/HypervisorHostHelper.java | 2 +- .../vmware/mo/PbmProfileManagerMO.java | 17 +++- .../vmware/util/VcenterSessionHandler.java | 88 +++++++++++++++++++ .../hypervisor/vmware/util/VmwareClient.java | 41 ++++++++- 4 files changed, 145 insertions(+), 3 deletions(-) create mode 100644 vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VcenterSessionHandler.java diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java index 1242d25f90c..63005406b79 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java @@ -1745,7 +1745,7 @@ public class HypervisorHostHelper { return ovfString; } - public static void importVmFromOVF(VmwareHypervisorHost host, String ovfFilePath, String vmName, DatastoreMO dsMo, String diskOption, ManagedObjectReference morRp, + public static void importVmFromOVF(VmwareHypervisorHost host, String ovfFilePath, String vmName, DatastoreMO dsMo, String diskOption, ManagedObjectReference morRp, ManagedObjectReference morHost) throws Exception { assert (morRp != null); diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/PbmProfileManagerMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/PbmProfileManagerMO.java index 352cbf30027..31a7a44deea 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/PbmProfileManagerMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/PbmProfileManagerMO.java @@ -1,6 +1,21 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. package com.cloud.hypervisor.vmware.mo; - import com.cloud.hypervisor.vmware.util.VmwareContext; import com.vmware.pbm.PbmProfile; diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VcenterSessionHandler.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VcenterSessionHandler.java new file mode 100644 index 00000000000..9efab7b8ece --- /dev/null +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VcenterSessionHandler.java @@ -0,0 +1,88 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.hypervisor.vmware.util; + +import java.util.Set; + +import javax.xml.namespace.QName; +import javax.xml.soap.SOAPElement; +import javax.xml.soap.SOAPException; +import javax.xml.soap.SOAPHeader; +import javax.xml.ws.handler.MessageContext; +import javax.xml.ws.handler.soap.SOAPHandler; +import javax.xml.ws.handler.soap.SOAPMessageContext; + +import org.apache.log4j.Logger; +import org.w3c.dom.DOMException; + +import com.cloud.utils.exception.CloudRuntimeException; + +public class VcenterSessionHandler implements SOAPHandler { + public static final Logger s_logger = Logger.getLogger(VcenterSessionHandler.class); + private final String vcSessionCookie; + + public VcenterSessionHandler(String vcSessionCookie) { + this.vcSessionCookie = vcSessionCookie; + } + + @Override + public boolean handleMessage(SOAPMessageContext smc) { + if (isOutgoingMessage(smc)) { + try { + SOAPHeader header = getSOAPHeader(smc); + + SOAPElement vcsessionHeader = header.addChildElement(new javax.xml.namespace.QName("#", + "vcSessionCookie")); + vcsessionHeader.setValue(vcSessionCookie); + + } catch (DOMException e) { + s_logger.debug(e); + throw new CloudRuntimeException(e); + } catch (SOAPException e) { + s_logger.debug(e); + throw new CloudRuntimeException(e); + } + } + return true; + } + + @Override + public void close(MessageContext arg0) { + } + + @Override + public boolean handleFault(SOAPMessageContext arg0) { + return false; + } + + @Override + public Set getHeaders() { + return null; + } + + SOAPHeader getSOAPHeader(SOAPMessageContext smc) throws SOAPException { + return smc.getMessage().getSOAPPart().getEnvelope().getHeader() == null ? smc + .getMessage().getSOAPPart().getEnvelope().addHeader() + : smc.getMessage().getSOAPPart().getEnvelope().getHeader(); + } + + boolean isOutgoingMessage(SOAPMessageContext smc) { + Boolean outboundProperty = (Boolean)smc.get(MessageContext.MESSAGE_OUTBOUND_PROPERTY); + return outboundProperty; + } + +} diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClient.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClient.java index 376afb1dbe7..2395ccf5326 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClient.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClient.java @@ -17,8 +17,11 @@ package com.cloud.hypervisor.vmware.util; import java.lang.reflect.Method; +import java.net.URI; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.StringTokenizer; @@ -29,10 +32,15 @@ import javax.net.ssl.SSLSession; import javax.xml.ws.BindingProvider; import javax.xml.ws.WebServiceException; import javax.xml.ws.handler.MessageContext; +import javax.xml.ws.handler.Handler; +import javax.xml.ws.handler.HandlerResolver; +import javax.xml.ws.handler.PortInfo; + import org.apache.cloudstack.utils.security.SSLUtils; import org.apache.cloudstack.utils.security.SecureSSLSocketFactory; import com.vmware.pbm.PbmPortType; +import com.vmware.pbm.PbmService; import com.vmware.pbm.PbmServiceInstanceContent; import org.apache.log4j.Logger; import org.w3c.dom.Element; @@ -103,6 +111,7 @@ public class VmwareClient { HttpsURLConnection.setDefaultHostnameVerifier(hv); vimService = new VimService(); + pbmService = new PbmService(); } catch (Exception e) { s_logger.info("[ignored]" + "failed to trust all certificates blindly: ", e); @@ -125,6 +134,8 @@ public class VmwareClient { private final ManagedObjectReference pbmSvcInstRef = new ManagedObjectReference(); private static VimService vimService; + private static PbmService pbmService; + private PbmServiceInstanceContent pbmServiceContent; private VimPortType vimPort; private PbmPortType pbmPort; private static final String PBM_SERVICE_INSTANCE_TYPE = "PbmServiceInstance"; @@ -184,10 +195,38 @@ public class VmwareClient { cookieValue = tokenizer.nextToken(); String pathData = "$" + tokenizer.nextToken(); serviceCookie = "$Version=\"1\"; " + cookieValue + "; " + pathData; - + Map> map = new HashMap>(); + map.put("Cookie", Collections.singletonList(serviceCookie)); + ((BindingProvider)vimPort).getRequestContext().put(MessageContext.HTTP_REQUEST_HEADERS, map); + pbmConnect(url, cookieValue); isConnected = true; } + private void pbmConnect(String url, String cookieValue) throws Exception { + URI uri = new URI(url); + String pbmurl = "https://" + uri.getHost() + "/pbm"; + String[] tokens = cookieValue.split("="); + String extractedCookie = tokens[1]; + + HandlerResolver soapHandlerResolver = new HandlerResolver() { + @Override + public List getHandlerChain(PortInfo portInfo) { + VcenterSessionHandler VcSessionHandler = new VcenterSessionHandler(extractedCookie); + List handlerChain = new ArrayList(); + handlerChain.add((Handler)VcSessionHandler); + return handlerChain; + } + }; + pbmService.setHandlerResolver(soapHandlerResolver); + + pbmSvcInstRef.setType(PBM_SERVICE_INSTANCE_TYPE); + pbmSvcInstRef.setValue(PBM_SERVICE_INSTANCE_VALUE); + pbmPort = pbmService.getPbmPort(); + Map pbmCtxt = ((BindingProvider)pbmPort).getRequestContext(); + pbmCtxt.put(BindingProvider.SESSION_MAINTAIN_PROPERTY, true); + pbmCtxt.put(BindingProvider.ENDPOINT_ADDRESS_PROPERTY, pbmurl); + } + /** * Disconnects the user session. * From a72782f2e507d6d5c652ba2563aa7cda59fa257b Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Thu, 28 May 2020 16:02:35 +0530 Subject: [PATCH 003/164] Fix root folder issue on datastore --- deps/install-non-oss.sh | 1 + .../hypervisor/vmware/resource/VmwareResource.java | 4 +++- .../storage/resource/VmwareStorageLayoutHelper.java | 6 ++++++ .../com/cloud/hypervisor/vmware/mo/DatastoreMO.java | 8 +++++++- .../java/com/cloud/hypervisor/vmware/mo/HostMO.java | 6 +++++- .../hypervisor/vmware/mo/HypervisorHostHelper.java | 11 +++++++++++ 6 files changed, 33 insertions(+), 3 deletions(-) diff --git a/deps/install-non-oss.sh b/deps/install-non-oss.sh index f8a526305a8..f1c63c4a95f 100755 --- a/deps/install-non-oss.sh +++ b/deps/install-non-oss.sh @@ -40,4 +40,5 @@ mvn install:install-file -Dfile=vim25_65.jar -DgroupId=com.cloud.com.vmwa # From https://my.vmware.com/group/vmware/details?downloadGroup=WEBCLIENTSDK67U2&productId=742 mvn install:install-file -Dfile=vim25_67.jar -DgroupId=com.cloud.com.vmware -DartifactId=vmware-vim25 -Dversion=6.7 -Dpackaging=jar +# From https://my.vmware.com/group/vmware/get-download?downloadGroup=VS-MGMT-SDK65 mvn install:install-file -Dfile=pbm_65.jar -DgroupId=com.cloud.com.vmware -DartifactId=pbm -Dversion=6.5 -Dpackaging=jar diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 711020ab7f8..0e13c309081 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -4913,8 +4913,10 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa assert (morDatastore != null); - DatastoreSummary summary = new DatastoreMO(getServiceContext(), morDatastore).getSummary(); + DatastoreMO dsMo = new DatastoreMO(getServiceContext(), morDatastore); + HypervisorHostHelper.createBaseFolderInDatastore(dsMo, hyperHost); + DatastoreSummary summary = dsMo.getSummary(); long capacity = summary.getCapacity(); long available = summary.getFreeSpace(); diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageLayoutHelper.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageLayoutHelper.java index 9b2acbc5179..a422fedb25f 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageLayoutHelper.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageLayoutHelper.java @@ -23,6 +23,8 @@ import org.apache.log4j.Logger; import com.cloud.hypervisor.vmware.mo.DatacenterMO; import com.cloud.hypervisor.vmware.mo.DatastoreFile; import com.cloud.hypervisor.vmware.mo.DatastoreMO; +import com.cloud.hypervisor.vmware.mo.HypervisorHostHelper; + import com.cloud.utils.Pair; /** @@ -309,6 +311,10 @@ public class VmwareStorageLayoutHelper { } public static String getLegacyDatastorePathFromVmdkFileName(DatastoreMO dsMo, String vmdkFileName) throws Exception { + return String.format("[%s] %s/%s", dsMo.getName(), HypervisorHostHelper.VSPHERE_DATASTORE_BASE_FOLDER, vmdkFileName); + } + + public static String getDeprecatedLegacyDatastorePathFromVmdkFileName(DatastoreMO dsMo, String vmdkFileName) throws Exception { return String.format("[%s] %s", dsMo.getName(), vmdkFileName); } diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatastoreMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatastoreMO.java index 1e065a15daa..19ae41f6c77 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatastoreMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatastoreMO.java @@ -260,6 +260,12 @@ public class DatastoreMO extends BaseMO { if (!DatastoreFile.isFullDatastorePath(destFullPath)) destFullPath = String.format("[%s] %s", destDsName, destFilePath); + DatastoreMO srcDsMo = new DatastoreMO(_context, morDestDs); + if (!srcDsMo.fileExists(srcFullPath)) { + s_logger.error(String.format("Cannot move file to destination datastore due to file %s does not exists", srcFullPath)); + return false; + } + ManagedObjectReference morTask = _context.getService().moveDatastoreFileTask(morFileManager, srcFullPath, morSrcDc, destFullPath, morDestDc, forceOverwrite); boolean result = _context.getVimClient().waitForTask(morTask); @@ -267,7 +273,7 @@ public class DatastoreMO extends BaseMO { _context.waitForTaskProgressDone(morTask); return true; } else { - s_logger.error("VMware moveDatgastoreFile_Task failed due to " + TaskMO.getTaskFailureInfo(_context, morTask)); + s_logger.error("VMware moveDatastoreFile_Task failed due to " + TaskMO.getTaskFailureInfo(_context, morTask)); } return false; } diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HostMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HostMO.java index c248f7a63f1..100f56c9441 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HostMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HostMO.java @@ -842,6 +842,7 @@ public class HostMO extends BaseMO implements VmwareHypervisorHost { s_logger.trace("vCenter API trace - mountDatastore(). target MOR: " + _mor.getValue() + ", vmfs: " + vmfsDatastore + ", poolHost: " + poolHostAddress + ", poolHostPort: " + poolHostPort + ", poolPath: " + poolPath + ", poolUuid: " + poolUuid); + DatastoreMO dsMo = null; HostDatastoreSystemMO hostDatastoreSystemMo = getHostDatastoreSystemMO(); ManagedObjectReference morDatastore = hostDatastoreSystemMo.findDatastore(poolUuid); if (morDatastore == null) { @@ -868,6 +869,7 @@ public class HostMO extends BaseMO implements VmwareHypervisorHost { s_logger.trace("vCenter API trace - mountDatastore() done(failed)"); throw new Exception(msg); } + dsMo = new DatastoreMO(_context, morDatastore); } else { morDatastore = _context.getDatastoreMorByPath(poolPath); if (morDatastore == null) { @@ -879,11 +881,13 @@ public class HostMO extends BaseMO implements VmwareHypervisorHost { throw new Exception(msg); } - DatastoreMO dsMo = new DatastoreMO(_context, morDatastore); + dsMo = new DatastoreMO(_context, morDatastore); dsMo.setCustomFieldValue(CustomFieldConstants.CLOUD_UUID, poolUuid); } } + HypervisorHostHelper.createBaseFolderInDatastore(dsMo, this); + if (s_logger.isTraceEnabled()) s_logger.trace("vCenter API trace - mountDatastore() done(successfully)"); diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java index 63005406b79..21e4f4c9e7c 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java @@ -138,6 +138,7 @@ public class HypervisorHostHelper { private static final String UNTAGGED_VLAN_NAME = "untagged"; private static final String VMDK_PACK_DIR = "ova"; private static final String OVA_OPTION_KEY_BOOTDISK = "cloud.ova.bootdisk"; + public static final String VSPHERE_DATASTORE_BASE_FOLDER = ".cloudstack.base.folder"; public static VirtualMachineMO findVmFromObjectContent(VmwareContext context, ObjectContent[] ocs, String name, String instanceNameCustomField) { @@ -171,6 +172,7 @@ public class HypervisorHostHelper { if (morDs == null) morDs = hyperHost.findDatastore(uuidName); + DatastoreMO dsMo = new DatastoreMO(hyperHost.getContext(), morDs); return morDs; } @@ -2084,4 +2086,13 @@ public class HypervisorHostHelper { return DiskControllerType.getType(controller) == DiskControllerType.ide; } + public static void createBaseFolderInDatastore(DatastoreMO dsMo, VmwareHypervisorHost hyperHost) throws Exception { + String dsPath = String.format("[%s]", dsMo.getName()); + String folderPath = String.format("[%s] %s", dsMo.getName(), VSPHERE_DATASTORE_BASE_FOLDER); + + if (!dsMo.folderExists(dsPath, VSPHERE_DATASTORE_BASE_FOLDER)) { + s_logger.info(String.format("vSphere datastore base folder: %s does not exist, now creating on datastore: %s", VSPHERE_DATASTORE_BASE_FOLDER, dsMo.getName())); + dsMo.makeDirectory(folderPath, hyperHost.getHyperHostDatacenter()); + } + } } From b8b4f0f9adaec90c8032067fbc0fb6f428a6cd4b Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Mon, 1 Jun 2020 11:17:33 +0530 Subject: [PATCH 004/164] Added importVsphereStoragePolicies API and scheme changes --- .../com/cloud/dc/VsphereStoragePolicy.java | 31 +++++ .../main/java/com/cloud/event/EventTypes.java | 5 + .../META-INF/db/schema-41400to41500.sql | 66 +++++++++++ .../vmware/VmwareDatacenterService.java | 3 + .../vmware/VsphereStoragePolicy.java | 31 +++++ .../vmware/VsphereStoragePolicyVO.java | 108 ++++++++++++++++++ .../vmware/dao/VsphereStoragePolicyDao.java | 23 ++++ .../dao/VsphereStoragePolicyDaoImpl.java | 23 ++++ .../vmware/manager/VmwareManagerImpl.java | 53 +++++++++ .../zone/ImportVsphereStoragePoliciesCmd.java | 106 +++++++++++++++++ .../ImportVsphereStoragePoliciesResponse.java | 89 +++++++++++++++ .../vmware/mo/PbmProfileManagerMO.java | 6 +- 12 files changed, 541 insertions(+), 3 deletions(-) create mode 100644 api/src/main/java/com/cloud/dc/VsphereStoragePolicy.java create mode 100644 plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VsphereStoragePolicy.java create mode 100644 plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VsphereStoragePolicyVO.java create mode 100644 plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/dao/VsphereStoragePolicyDao.java create mode 100644 plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/dao/VsphereStoragePolicyDaoImpl.java create mode 100644 plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ImportVsphereStoragePoliciesCmd.java create mode 100644 plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/response/ImportVsphereStoragePoliciesResponse.java diff --git a/api/src/main/java/com/cloud/dc/VsphereStoragePolicy.java b/api/src/main/java/com/cloud/dc/VsphereStoragePolicy.java new file mode 100644 index 00000000000..ca0ed544700 --- /dev/null +++ b/api/src/main/java/com/cloud/dc/VsphereStoragePolicy.java @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.dc; + +import org.apache.cloudstack.api.Identity; +import org.apache.cloudstack.api.InternalIdentity; + +public interface VsphereStoragePolicy extends Identity, InternalIdentity { + + long getZoneId(); + + String getPolicyId(); + + String getName(); + + String getDescription(); +} diff --git a/api/src/main/java/com/cloud/event/EventTypes.java b/api/src/main/java/com/cloud/event/EventTypes.java index d723f563ad8..852198b91a7 100644 --- a/api/src/main/java/com/cloud/event/EventTypes.java +++ b/api/src/main/java/com/cloud/event/EventTypes.java @@ -619,6 +619,9 @@ public class EventTypes { public static final String EVENT_POD_ROLLING_MAINTENANCE = "POD.ROLLING.MAINTENANCE"; public static final String EVENT_ZONE_ROLLING_MAINTENANCE = "ZONE.ROLLING.MAINTENANCE"; + // Storage Policies + public static final String EVENT_IMPORT_VCENTER_STORAGE_POLICIES = "IMPORT.VCENTER.STORAGE.POLICIES"; + static { // TODO: need a way to force author adding event types to declare the entity details as well, with out braking @@ -1026,6 +1029,8 @@ public class EventTypes { entityEventDetails.put(EVENT_CLUSTER_ROLLING_MAINTENANCE, ClusterResponse.class); entityEventDetails.put(EVENT_HOST_ROLLING_MAINTENANCE, HostResponse.class); + entityEventDetails.put(EVENT_IMPORT_VCENTER_STORAGE_POLICIES, "StoragePolicies"); + entityEventDetails.put(EVENT_IMAGE_STORE_DATA_MIGRATE, ImageStore.class); } diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql b/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql index 20ed35c1f5b..4c7c7c994da 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql @@ -195,6 +195,72 @@ INSERT IGNORE INTO `cloud`.`hypervisor_capabilities`(uuid, hypervisor_type, hype -- Copy XenServer 8.0 hypervisor guest OS mappings to XenServer8.1 INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) SELECT UUID(),'Xenserver', '8.1.0', guest_os_name, guest_os_id, utc_timestamp(), 0 FROM `cloud`.`guest_os_hypervisor` WHERE hypervisor_type='Xenserver' AND hypervisor_version='8.0.0'; +CREATE TABLE IF NOT EXISTS `cloud`.`vsphere_storage_policy` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT, + `uuid` varchar(255) UNIQUE, + `zone_id` bigint(20) unsigned NOT NULL COMMENT 'id of the zone', + `policy_id` varchar(255) NOT NULL COMMENT 'the identifier of the Storage Policy in vSphere DataCenter', + `name` varchar(255) NOT NULL COMMENT 'name of the storage policy', + `description` text COMMENT 'description of the storage policy', + `update_time` datetime COMMENT 'last updated when policy imported', + `removed` datetime COMMENT 'date removed', + PRIMARY KEY (`id`), + KEY `fk_vsphere_storage_policy__zone_id` (`zone_id`), + UNIQUE KEY (`zone_id`, `policy_id`), + CONSTRAINT `fk_vsphere_storage_policy__zone_id` FOREIGN KEY (`zone_id`) REFERENCES `data_center` (`id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +ALTER TABLE `cloud`.`storage_pool` ADD COLUMN `parent` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'ID of the Datastore cluster (storage pool) if this is a child in that Datastore cluster'; + +-- Added parent column to support datastore clusters in vmware vsphere +DROP VIEW IF EXISTS `cloud`.`storage_pool_view`; +CREATE VIEW `cloud`.`storage_pool_view` AS + SELECT + `storage_pool`.`id` AS `id`, + `storage_pool`.`uuid` AS `uuid`, + `storage_pool`.`name` AS `name`, + `storage_pool`.`status` AS `status`, + `storage_pool`.`path` AS `path`, + `storage_pool`.`pool_type` AS `pool_type`, + `storage_pool`.`host_address` AS `host_address`, + `storage_pool`.`created` AS `created`, + `storage_pool`.`removed` AS `removed`, + `storage_pool`.`capacity_bytes` AS `capacity_bytes`, + `storage_pool`.`capacity_iops` AS `capacity_iops`, + `storage_pool`.`scope` AS `scope`, + `storage_pool`.`hypervisor` AS `hypervisor`, + `storage_pool`.`storage_provider_name` AS `storage_provider_name`, + `storage_pool`.`parent` AS `parent`, + `cluster`.`id` AS `cluster_id`, + `cluster`.`uuid` AS `cluster_uuid`, + `cluster`.`name` AS `cluster_name`, + `cluster`.`cluster_type` AS `cluster_type`, + `data_center`.`id` AS `data_center_id`, + `data_center`.`uuid` AS `data_center_uuid`, + `data_center`.`name` AS `data_center_name`, + `data_center`.`networktype` AS `data_center_type`, + `host_pod_ref`.`id` AS `pod_id`, + `host_pod_ref`.`uuid` AS `pod_uuid`, + `host_pod_ref`.`name` AS `pod_name`, + `storage_pool_tags`.`tag` AS `tag`, + `op_host_capacity`.`used_capacity` AS `disk_used_capacity`, + `op_host_capacity`.`reserved_capacity` AS `disk_reserved_capacity`, + `async_job`.`id` AS `job_id`, + `async_job`.`uuid` AS `job_uuid`, + `async_job`.`job_status` AS `job_status`, + `async_job`.`account_id` AS `job_account_id` + FROM + ((((((`storage_pool` + LEFT JOIN `cluster` ON ((`storage_pool`.`cluster_id` = `cluster`.`id`))) + LEFT JOIN `data_center` ON ((`storage_pool`.`data_center_id` = `data_center`.`id`))) + LEFT JOIN `host_pod_ref` ON ((`storage_pool`.`pod_id` = `host_pod_ref`.`id`))) + LEFT JOIN `storage_pool_tags` ON (((`storage_pool_tags`.`pool_id` = `storage_pool`.`id`)))) + LEFT JOIN `op_host_capacity` ON (((`storage_pool`.`id` = `op_host_capacity`.`host_id`) + AND (`op_host_capacity`.`capacity_type` IN (3 , 9))))) + LEFT JOIN `async_job` ON (((`async_job`.`instance_id` = `storage_pool`.`id`) + AND (`async_job`.`instance_type` = 'StoragePool') + AND (`async_job`.`job_status` = 0)))); + ALTER TABLE `cloud`.`image_store` ADD COLUMN `readonly` boolean DEFAULT false COMMENT 'defines status of image store'; ALTER VIEW `cloud`.`image_store_view` AS diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenterService.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenterService.java index 53792539ee8..7810cb38632 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenterService.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenterService.java @@ -20,6 +20,7 @@ package com.cloud.hypervisor.vmware; import java.util.List; import org.apache.cloudstack.api.command.admin.zone.AddVmwareDcCmd; +import org.apache.cloudstack.api.command.admin.zone.ImportVsphereStoragePoliciesCmd; import org.apache.cloudstack.api.command.admin.zone.ListVmwareDcsCmd; import org.apache.cloudstack.api.command.admin.zone.RemoveVmwareDcCmd; import org.apache.cloudstack.api.command.admin.zone.UpdateVmwareDcCmd; @@ -38,4 +39,6 @@ public interface VmwareDatacenterService extends PluggableService { boolean removeVmwareDatacenter(RemoveVmwareDcCmd cmd) throws IllegalArgumentException, ResourceInUseException; List listVmwareDatacenters(ListVmwareDcsCmd cmd) throws IllegalArgumentException, CloudRuntimeException; + + List importVsphereStoragePolicies(ImportVsphereStoragePoliciesCmd cmd); } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VsphereStoragePolicy.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VsphereStoragePolicy.java new file mode 100644 index 00000000000..102bdaa3c04 --- /dev/null +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VsphereStoragePolicy.java @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.hypervisor.vmware; + +import org.apache.cloudstack.api.Identity; +import org.apache.cloudstack.api.InternalIdentity; + +public interface VsphereStoragePolicy extends Identity, InternalIdentity { + + long getZoneId(); + + String getPolicyId(); + + String getName(); + + String getDescription(); +} diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VsphereStoragePolicyVO.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VsphereStoragePolicyVO.java new file mode 100644 index 00000000000..bc07b886719 --- /dev/null +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VsphereStoragePolicyVO.java @@ -0,0 +1,108 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.hypervisor.vmware; + +import com.cloud.utils.db.GenericDao; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; +import javax.persistence.Temporal; +import javax.persistence.TemporalType; +import java.util.Date; +import java.util.UUID; + +@Entity +@Table(name = "vsphere_storage_policy") +public class VsphereStoragePolicyVO implements VsphereStoragePolicy { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + private long id; + + @Column(name = "uuid") + private String uuid; + + @Column(name = "zone_id") + private long zoneId; + + @Column(name = "policy_id") + private String policyId; + + @Column(name = "name") + private String name; + + @Column(name = "description") + private String description; + + @Column(name = "update_time", updatable = true) + @Temporal(value = TemporalType.TIMESTAMP) + private Date updateTime; + + @Column(name = GenericDao.REMOVED_COLUMN) + private Date removed; + + public VsphereStoragePolicyVO(long zoneId, String policyId, String name, String description) { + this.uuid = UUID.randomUUID().toString(); + this.zoneId = zoneId; + this.policyId = policyId; + this.name = name; + this.description = description; + } + + @Override + public long getId() { + return id; + } + + @Override + public String getUuid() { + return uuid; + } + + @Override + public long getZoneId() { + return zoneId; + } + + @Override + public String getPolicyId() { + return policyId; + } + + @Override + public String getName() { + return name; + } + + @Override + public String getDescription() { + return description; + } + + public Date getUpdateTime() { + return updateTime; + } + + public Date getRemoved() { + return removed; + } +} diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/dao/VsphereStoragePolicyDao.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/dao/VsphereStoragePolicyDao.java new file mode 100644 index 00000000000..14f77af9bd8 --- /dev/null +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/dao/VsphereStoragePolicyDao.java @@ -0,0 +1,23 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.hypervisor.vmware.dao; + +import com.cloud.hypervisor.vmware.VsphereStoragePolicyVO; +import com.cloud.utils.db.GenericDao; + +public interface VsphereStoragePolicyDao extends GenericDao { +} diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/dao/VsphereStoragePolicyDaoImpl.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/dao/VsphereStoragePolicyDaoImpl.java new file mode 100644 index 00000000000..2d13eff0c5b --- /dev/null +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/dao/VsphereStoragePolicyDaoImpl.java @@ -0,0 +1,23 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.hypervisor.vmware.dao; + +import com.cloud.hypervisor.vmware.VsphereStoragePolicyVO; +import com.cloud.utils.db.GenericDaoBase; + +public class VsphereStoragePolicyDaoImpl extends GenericDaoBase implements VsphereStoragePolicyDao { +} diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java index 9cbaaf7a55c..e18d837c4a3 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java @@ -38,7 +38,13 @@ import java.util.concurrent.TimeUnit; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.hypervisor.vmware.VsphereStoragePolicy; +import com.cloud.hypervisor.vmware.VsphereStoragePolicyVO; +import com.cloud.hypervisor.vmware.dao.VsphereStoragePolicyDao; +import com.cloud.hypervisor.vmware.mo.PbmProfileManagerMO; +import com.vmware.pbm.PbmProfile; import org.apache.cloudstack.api.command.admin.zone.AddVmwareDcCmd; +import org.apache.cloudstack.api.command.admin.zone.ImportVsphereStoragePoliciesCmd; import org.apache.cloudstack.api.command.admin.zone.ListVmwareDcsCmd; import org.apache.cloudstack.api.command.admin.zone.RemoveVmwareDcCmd; import org.apache.cloudstack.api.command.admin.zone.UpdateVmwareDcCmd; @@ -208,6 +214,8 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw private UserVmCloneSettingDao cloneSettingDao; @Inject private TemplateManager templateManager; + @Inject + private VsphereStoragePolicyDao vsphereStoragePolicyDao; private String _mountParent; private StorageLayer _storage; @@ -1383,6 +1391,51 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw } } + @Override + public List importVsphereStoragePolicies(ImportVsphereStoragePoliciesCmd cmd) { + Long zoneId = cmd.getZoneId(); + // Validate Id of zone + doesZoneExist(zoneId); + + // Get DC associated with this zone + VmwareDatacenterVO vmwareDatacenter; + String vmwareDcName; + String vCenterHost; + String userName; + String password; + DatacenterMO dcMo = null; + final VmwareDatacenterZoneMapVO vmwareDcZoneMap = vmwareDatacenterZoneMapDao.findByZoneId(zoneId); + // Check if zone is associated with VMware DC + if (vmwareDcZoneMap == null) { + throw new CloudRuntimeException("Zone " + zoneId + " is not associated with any VMware datacenter."); + } + + final long vmwareDcId = vmwareDcZoneMap.getVmwareDcId(); + vmwareDatacenter = vmwareDcDao.findById(vmwareDcId); + vmwareDcName = vmwareDatacenter.getVmwareDatacenterName(); + vCenterHost = vmwareDatacenter.getVcenterHost(); + userName = vmwareDatacenter.getUser(); + password = vmwareDatacenter.getPassword(); + List storageProfiles = null; + try { + VmwareContext context = VmwareContextFactory.getContext(vCenterHost, userName, password); + PbmProfileManagerMO profileManagerMO = new PbmProfileManagerMO(context); + storageProfiles = profileManagerMO.getStorageProfiles(); + } catch (Exception e) { + String msg = String.format("Unable to list storage profiles from DC %s due to : %s", vmwareDcName, VmwareHelper.getExceptionMessage(e); + s_logger.error(msg); + throw new CloudRuntimeException(msg); + } + + for (PbmProfile storageProfile : storageProfiles) { + VsphereStoragePolicyVO StoragePolicyVO = new VsphereStoragePolicyVO(zoneId, storageProfile.getProfileId().toString(), storageProfile.getName(), storageProfile.getDescription()); + vsphereStoragePolicyDao.persist(StoragePolicyVO); + } + + List storagePolicies = vsphereStoragePolicyDao.listAll(); + return storagePolicies; + } + @Override public boolean hasNexusVSM(Long clusterId) { ClusterVSMMapVO vsmMapVo = null; diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ImportVsphereStoragePoliciesCmd.java b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ImportVsphereStoragePoliciesCmd.java new file mode 100644 index 00000000000..cff37f50d3f --- /dev/null +++ b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ImportVsphereStoragePoliciesCmd.java @@ -0,0 +1,106 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.admin.zone; + +import com.cloud.event.EventTypes; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.hypervisor.vmware.VmwareDatacenterService; +import com.cloud.hypervisor.vmware.VsphereStoragePolicy; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.ImportVsphereStoragePoliciesResponse; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.RoleResponse; +import org.apache.cloudstack.api.response.ZoneResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.log4j.Logger; + +import javax.inject.Inject; +import java.util.ArrayList; +import java.util.List; + +@APICommand(name = ImportVsphereStoragePoliciesCmd.APINAME, description = "Import vSphere storage policies", + responseObject = ImportVsphereStoragePoliciesResponse.class, + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, + authorized = {RoleType.Admin}) +public class ImportVsphereStoragePoliciesCmd extends BaseAsyncCmd { + + public static final Logger s_logger = Logger.getLogger(ImportVsphereStoragePoliciesCmd.class.getName()); + + public static final String APINAME = "importVsphereStoragePolicies"; + + @Inject + public VmwareDatacenterService _vmwareDatacenterService; + + @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.UUID, entityType = ZoneResponse.class, + description = "ID of the zone") + private Long zoneId; + + @Override + public String getEventType() { + return EventTypes.EVENT_IMPORT_VCENTER_STORAGE_POLICIES; + } + + @Override + public String getEventDescription() { + return "Importing vSphere Storage Policies"; + } + + @Override + public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { + List storagePolicies = _vmwareDatacenterService.importVsphereStoragePolicies(this); + final ListResponse responseList = new ListResponse<>(); + final List storagePoliciesResponseList = new ArrayList<>(); + for (VsphereStoragePolicy storagePolicy : storagePolicies) { + final ImportVsphereStoragePoliciesResponse storagePoliciesResponse = new ImportVsphereStoragePoliciesResponse(); + storagePoliciesResponse.setId(storagePolicy.getUuid()); + storagePoliciesResponse.setName(storagePolicy.getName()); + storagePoliciesResponse.setPolicyId(storagePolicy.getPolicyId()); + storagePoliciesResponse.setDescription(storagePolicy.getDescription()); + storagePoliciesResponseList.add(storagePoliciesResponse); + } + responseList.setResponses(storagePoliciesResponseList); + responseList.setResponseName(getCommandName()); + setResponseObject(responseList); + } + + @Override + public String getCommandName() { + return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX; + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccountId(); + } + + public Long getZoneId() { + return zoneId; + } + +} diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/response/ImportVsphereStoragePoliciesResponse.java b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/response/ImportVsphereStoragePoliciesResponse.java new file mode 100644 index 00000000000..82baf080348 --- /dev/null +++ b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/response/ImportVsphereStoragePoliciesResponse.java @@ -0,0 +1,89 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.response; + +import com.cloud.hypervisor.vmware.VsphereStoragePolicy; +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; +import org.apache.cloudstack.api.EntityReference; + + +@EntityReference(value = VsphereStoragePolicy.class) +public class ImportVsphereStoragePoliciesResponse extends BaseResponse { + + @SerializedName(ApiConstants.ID) + @Param(description = "the ID of the Storage Policy") + private String id; + + @SerializedName(ApiConstants.ZONE_ID) + @Param(description = "the ID of the Zone") + private String zoneId; + + @SerializedName(ApiConstants.POLICY_ID) + @Param(description = "the identifier of the Storage Policy in vSphere DataCenter") + private String policyId; + + @SerializedName(ApiConstants.NAME) + @Param(description = "the name of the Storage Policy") + private String name; + + @SerializedName(ApiConstants.DESCRIPTION) + @Param(description = "the description of the Storage Policy") + private String description; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getZoneId() { + return zoneId; + } + + public void setZoneId(String zoneId) { + this.zoneId = zoneId; + } + + public String getPolicyId() { + return policyId; + } + + public void setPolicyId(String policyId) { + this.policyId = policyId; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } +} diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/PbmProfileManagerMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/PbmProfileManagerMO.java index 31a7a44deea..9109b18f7b3 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/PbmProfileManagerMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/PbmProfileManagerMO.java @@ -44,7 +44,7 @@ public class PbmProfileManagerMO extends BaseMO { super(context, morType, morValue); } - public List getProfileIds() throws Exception { + public List getStorageProfileIds() throws Exception { if (s_logger.isDebugEnabled()) { s_logger.debug("Querying vCenter " + _context.getServerAddress() + " for profiles"); } @@ -52,8 +52,8 @@ public class PbmProfileManagerMO extends BaseMO { return profileIds; } - public List getProfiles(PbmProfileResourceType pbmResourceType) throws Exception { - List profileIds = getProfileIds(); + public List getStorageProfiles() throws Exception { + List profileIds = getStorageProfileIds(); List profiles = _context.getPbmService().pbmRetrieveContent(_mor, profileIds); return profiles; } From 037fcfd03221ef87a1bcb817fea01929bf3a8a04 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Tue, 2 Jun 2020 21:14:27 +0530 Subject: [PATCH 005/164] Fixed bean creation errors on VsphereStoragePolicyDaoImpl --- .../dao/VsphereStoragePolicyDaoImpl.java | 22 +++++++++++++++++++ .../vmware/manager/VmwareManagerImpl.java | 3 ++- .../zone/ImportVsphereStoragePoliciesCmd.java | 1 - .../core/spring-vmware-core-context.xml | 2 +- 4 files changed, 25 insertions(+), 3 deletions(-) diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/dao/VsphereStoragePolicyDaoImpl.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/dao/VsphereStoragePolicyDaoImpl.java index 2d13eff0c5b..291daae11f8 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/dao/VsphereStoragePolicyDaoImpl.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/dao/VsphereStoragePolicyDaoImpl.java @@ -18,6 +18,28 @@ package com.cloud.hypervisor.vmware.dao; import com.cloud.hypervisor.vmware.VsphereStoragePolicyVO; import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; +@Component public class VsphereStoragePolicyDaoImpl extends GenericDaoBase implements VsphereStoragePolicyDao { + + protected static final Logger LOGGER = Logger.getLogger(VsphereStoragePolicyDaoImpl.class); + + private final SearchBuilder zoneSearch; + private final SearchBuilder policySearch; + + public VsphereStoragePolicyDaoImpl() { + super(); + + zoneSearch = createSearchBuilder(); + zoneSearch.and("zoneId", zoneSearch.entity().getZoneId(), SearchCriteria.Op.EQ); + zoneSearch.done(); + + policySearch = createSearchBuilder(); + policySearch.and("policyId", policySearch.entity().getPolicyId(), SearchCriteria.Op.EQ); + policySearch.done(); + } } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java index e18d837c4a3..d489364aaaa 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java @@ -1054,6 +1054,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw cmdList.add(UpdateVmwareDcCmd.class); cmdList.add(RemoveVmwareDcCmd.class); cmdList.add(ListVmwareDcsCmd.class); + cmdList.add(ImportVsphereStoragePoliciesCmd.class); return cmdList; } @@ -1422,7 +1423,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw PbmProfileManagerMO profileManagerMO = new PbmProfileManagerMO(context); storageProfiles = profileManagerMO.getStorageProfiles(); } catch (Exception e) { - String msg = String.format("Unable to list storage profiles from DC %s due to : %s", vmwareDcName, VmwareHelper.getExceptionMessage(e); + String msg = String.format("Unable to list storage profiles from DC %s due to : %s", vmwareDcName, VmwareHelper.getExceptionMessage(e)); s_logger.error(msg); throw new CloudRuntimeException(msg); } diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ImportVsphereStoragePoliciesCmd.java b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ImportVsphereStoragePoliciesCmd.java index cff37f50d3f..6938588a994 100644 --- a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ImportVsphereStoragePoliciesCmd.java +++ b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ImportVsphereStoragePoliciesCmd.java @@ -35,7 +35,6 @@ import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.response.ImportVsphereStoragePoliciesResponse; import org.apache.cloudstack.api.response.ListResponse; -import org.apache.cloudstack.api.response.RoleResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.cloudstack.context.CallContext; import org.apache.log4j.Logger; diff --git a/plugins/hypervisors/vmware/src/main/resources/META-INF/cloudstack/core/spring-vmware-core-context.xml b/plugins/hypervisors/vmware/src/main/resources/META-INF/cloudstack/core/spring-vmware-core-context.xml index 3af2d1ac31f..406a6976b9a 100644 --- a/plugins/hypervisors/vmware/src/main/resources/META-INF/cloudstack/core/spring-vmware-core-context.xml +++ b/plugins/hypervisors/vmware/src/main/resources/META-INF/cloudstack/core/spring-vmware-core-context.xml @@ -36,7 +36,7 @@ - + From a4ec86a652dac8ca5d91cd7aa8b364123acd9627 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Wed, 3 Jun 2020 12:56:19 +0530 Subject: [PATCH 006/164] Added Storage policy id to VO --- .../hypervisor/vmware/VsphereStoragePolicyVO.java | 10 ++++++++++ .../hypervisor/vmware/manager/VmwareManagerImpl.java | 2 +- .../admin/zone/ImportVsphereStoragePoliciesCmd.java | 10 ++++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VsphereStoragePolicyVO.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VsphereStoragePolicyVO.java index bc07b886719..13c46784f19 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VsphereStoragePolicyVO.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VsphereStoragePolicyVO.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.hypervisor.vmware; +import com.cloud.utils.DateUtil; import com.cloud.utils.db.GenericDao; import javax.persistence.Column; @@ -66,6 +67,15 @@ public class VsphereStoragePolicyVO implements VsphereStoragePolicy { this.policyId = policyId; this.name = name; this.description = description; + this.updateTime = DateUtil.currentGMTTime(); + } + + public VsphereStoragePolicyVO() { + uuid = UUID.randomUUID().toString(); + } + public VsphereStoragePolicyVO(long id) { + this.id = id; + uuid = UUID.randomUUID().toString(); } @Override diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java index d489364aaaa..c25189786b5 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java @@ -1429,7 +1429,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw } for (PbmProfile storageProfile : storageProfiles) { - VsphereStoragePolicyVO StoragePolicyVO = new VsphereStoragePolicyVO(zoneId, storageProfile.getProfileId().toString(), storageProfile.getName(), storageProfile.getDescription()); + VsphereStoragePolicyVO StoragePolicyVO = new VsphereStoragePolicyVO(zoneId, storageProfile.getProfileId().getUniqueId(), storageProfile.getName(), storageProfile.getDescription()); vsphereStoragePolicyDao.persist(StoragePolicyVO); } diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ImportVsphereStoragePoliciesCmd.java b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ImportVsphereStoragePoliciesCmd.java index 6938588a994..b0e1d82a02e 100644 --- a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ImportVsphereStoragePoliciesCmd.java +++ b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ImportVsphereStoragePoliciesCmd.java @@ -17,6 +17,7 @@ package org.apache.cloudstack.api.command.admin.zone; +import com.cloud.dc.DataCenter; import com.cloud.event.EventTypes; import com.cloud.exception.ConcurrentOperationException; import com.cloud.exception.InsufficientCapacityException; @@ -29,6 +30,7 @@ import com.cloud.hypervisor.vmware.VsphereStoragePolicy; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseAsyncCmd; import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.Parameter; @@ -72,15 +74,23 @@ public class ImportVsphereStoragePoliciesCmd extends BaseAsyncCmd { @Override public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { + final DataCenter dataCenter = _resourceService.getZone(getZoneId()); + if (dataCenter == null) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Unable to find zone by ID: " + getZoneId()); + } + List storagePolicies = _vmwareDatacenterService.importVsphereStoragePolicies(this); final ListResponse responseList = new ListResponse<>(); final List storagePoliciesResponseList = new ArrayList<>(); for (VsphereStoragePolicy storagePolicy : storagePolicies) { final ImportVsphereStoragePoliciesResponse storagePoliciesResponse = new ImportVsphereStoragePoliciesResponse(); + storagePoliciesResponse.setZoneId(dataCenter.getUuid()); storagePoliciesResponse.setId(storagePolicy.getUuid()); storagePoliciesResponse.setName(storagePolicy.getName()); storagePoliciesResponse.setPolicyId(storagePolicy.getPolicyId()); storagePoliciesResponse.setDescription(storagePolicy.getDescription()); + storagePoliciesResponse.setObjectName("StoragePolicy"); + storagePoliciesResponseList.add(storagePoliciesResponse); } responseList.setResponses(storagePoliciesResponseList); From 8b23ea90b8566b0f9e5ae7b6a17b0bf2f58728b5 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Thu, 4 Jun 2020 12:00:07 +0530 Subject: [PATCH 007/164] Moved Dao and VO files to different packages --- .../com/cloud/dc}/VsphereStoragePolicyVO.java | 2 +- .../dc}/dao/VsphereStoragePolicyDao.java | 4 +-- .../dc}/dao/VsphereStoragePolicyDaoImpl.java | 4 +-- ...spring-engine-schema-core-daos-context.xml | 1 + .../vmware/VmwareDatacenterService.java | 1 + .../vmware/VsphereStoragePolicy.java | 31 ------------------- .../vmware/manager/VmwareManagerImpl.java | 6 ++-- .../zone/ImportVsphereStoragePoliciesCmd.java | 2 +- .../ImportVsphereStoragePoliciesResponse.java | 2 +- .../core/spring-vmware-core-context.xml | 1 - 10 files changed, 12 insertions(+), 42 deletions(-) rename {plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware => engine/schema/src/main/java/com/cloud/dc}/VsphereStoragePolicyVO.java (98%) rename {plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware => engine/schema/src/main/java/com/cloud/dc}/dao/VsphereStoragePolicyDao.java (90%) rename {plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware => engine/schema/src/main/java/com/cloud/dc}/dao/VsphereStoragePolicyDaoImpl.java (94%) delete mode 100644 plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VsphereStoragePolicy.java diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VsphereStoragePolicyVO.java b/engine/schema/src/main/java/com/cloud/dc/VsphereStoragePolicyVO.java similarity index 98% rename from plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VsphereStoragePolicyVO.java rename to engine/schema/src/main/java/com/cloud/dc/VsphereStoragePolicyVO.java index 13c46784f19..591455ed8a6 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VsphereStoragePolicyVO.java +++ b/engine/schema/src/main/java/com/cloud/dc/VsphereStoragePolicyVO.java @@ -14,7 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -package com.cloud.hypervisor.vmware; +package com.cloud.dc;; import com.cloud.utils.DateUtil; import com.cloud.utils.db.GenericDao; diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/dao/VsphereStoragePolicyDao.java b/engine/schema/src/main/java/com/cloud/dc/dao/VsphereStoragePolicyDao.java similarity index 90% rename from plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/dao/VsphereStoragePolicyDao.java rename to engine/schema/src/main/java/com/cloud/dc/dao/VsphereStoragePolicyDao.java index 14f77af9bd8..907ab201818 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/dao/VsphereStoragePolicyDao.java +++ b/engine/schema/src/main/java/com/cloud/dc/dao/VsphereStoragePolicyDao.java @@ -14,9 +14,9 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -package com.cloud.hypervisor.vmware.dao; +package com.cloud.dc.dao; -import com.cloud.hypervisor.vmware.VsphereStoragePolicyVO; +import com.cloud.dc.VsphereStoragePolicyVO; import com.cloud.utils.db.GenericDao; public interface VsphereStoragePolicyDao extends GenericDao { diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/dao/VsphereStoragePolicyDaoImpl.java b/engine/schema/src/main/java/com/cloud/dc/dao/VsphereStoragePolicyDaoImpl.java similarity index 94% rename from plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/dao/VsphereStoragePolicyDaoImpl.java rename to engine/schema/src/main/java/com/cloud/dc/dao/VsphereStoragePolicyDaoImpl.java index 291daae11f8..4b2fd234132 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/dao/VsphereStoragePolicyDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/dc/dao/VsphereStoragePolicyDaoImpl.java @@ -14,9 +14,9 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -package com.cloud.hypervisor.vmware.dao; +package com.cloud.dc.dao; -import com.cloud.hypervisor.vmware.VsphereStoragePolicyVO; +import com.cloud.dc.VsphereStoragePolicyVO; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; diff --git a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml index 9ab5774b301..052a3dfc205 100644 --- a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml +++ b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml @@ -295,4 +295,5 @@ + diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenterService.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenterService.java index 7810cb38632..6e69009b62f 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenterService.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenterService.java @@ -19,6 +19,7 @@ package com.cloud.hypervisor.vmware; import java.util.List; +import com.cloud.dc.VsphereStoragePolicy; import org.apache.cloudstack.api.command.admin.zone.AddVmwareDcCmd; import org.apache.cloudstack.api.command.admin.zone.ImportVsphereStoragePoliciesCmd; import org.apache.cloudstack.api.command.admin.zone.ListVmwareDcsCmd; diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VsphereStoragePolicy.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VsphereStoragePolicy.java deleted file mode 100644 index 102bdaa3c04..00000000000 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VsphereStoragePolicy.java +++ /dev/null @@ -1,31 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -package com.cloud.hypervisor.vmware; - -import org.apache.cloudstack.api.Identity; -import org.apache.cloudstack.api.InternalIdentity; - -public interface VsphereStoragePolicy extends Identity, InternalIdentity { - - long getZoneId(); - - String getPolicyId(); - - String getName(); - - String getDescription(); -} diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java index c25189786b5..85b7beeec69 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java @@ -38,9 +38,9 @@ import java.util.concurrent.TimeUnit; import javax.inject.Inject; import javax.naming.ConfigurationException; -import com.cloud.hypervisor.vmware.VsphereStoragePolicy; -import com.cloud.hypervisor.vmware.VsphereStoragePolicyVO; -import com.cloud.hypervisor.vmware.dao.VsphereStoragePolicyDao; +import com.cloud.dc.VsphereStoragePolicy; +import com.cloud.dc.VsphereStoragePolicyVO; +import com.cloud.dc.dao.VsphereStoragePolicyDao; import com.cloud.hypervisor.vmware.mo.PbmProfileManagerMO; import com.vmware.pbm.PbmProfile; import org.apache.cloudstack.api.command.admin.zone.AddVmwareDcCmd; diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ImportVsphereStoragePoliciesCmd.java b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ImportVsphereStoragePoliciesCmd.java index b0e1d82a02e..cf950003d5b 100644 --- a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ImportVsphereStoragePoliciesCmd.java +++ b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ImportVsphereStoragePoliciesCmd.java @@ -18,6 +18,7 @@ package org.apache.cloudstack.api.command.admin.zone; import com.cloud.dc.DataCenter; +import com.cloud.dc.VsphereStoragePolicy; import com.cloud.event.EventTypes; import com.cloud.exception.ConcurrentOperationException; import com.cloud.exception.InsufficientCapacityException; @@ -25,7 +26,6 @@ import com.cloud.exception.NetworkRuleConflictException; import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; import com.cloud.hypervisor.vmware.VmwareDatacenterService; -import com.cloud.hypervisor.vmware.VsphereStoragePolicy; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.APICommand; diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/response/ImportVsphereStoragePoliciesResponse.java b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/response/ImportVsphereStoragePoliciesResponse.java index 82baf080348..a2227fb8e7f 100644 --- a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/response/ImportVsphereStoragePoliciesResponse.java +++ b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/response/ImportVsphereStoragePoliciesResponse.java @@ -16,7 +16,7 @@ // under the License. package org.apache.cloudstack.api.response; -import com.cloud.hypervisor.vmware.VsphereStoragePolicy; +import com.cloud.dc.VsphereStoragePolicy; import com.cloud.serializer.Param; import com.google.gson.annotations.SerializedName; import org.apache.cloudstack.api.ApiConstants; diff --git a/plugins/hypervisors/vmware/src/main/resources/META-INF/cloudstack/core/spring-vmware-core-context.xml b/plugins/hypervisors/vmware/src/main/resources/META-INF/cloudstack/core/spring-vmware-core-context.xml index 406a6976b9a..a2d8314bfb8 100644 --- a/plugins/hypervisors/vmware/src/main/resources/META-INF/cloudstack/core/spring-vmware-core-context.xml +++ b/plugins/hypervisors/vmware/src/main/resources/META-INF/cloudstack/core/spring-vmware-core-context.xml @@ -36,7 +36,6 @@ - From 2ff4989feb6d48ffaa4ed74b00425bff8ecd9cae Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Thu, 4 Jun 2020 15:31:34 +0530 Subject: [PATCH 008/164] Added gen_toc.py changes for API importvpsherestoragepolicies --- tools/apidoc/gen_toc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/apidoc/gen_toc.py b/tools/apidoc/gen_toc.py index ca1c44fcd82..73b8e2e871e 100644 --- a/tools/apidoc/gen_toc.py +++ b/tools/apidoc/gen_toc.py @@ -196,7 +196,8 @@ known_categories = { 'KubernetesSupportedVersion': 'Kubernetes Service', 'KubernetesCluster': 'Kubernetes Service', 'UnmanagedInstance': 'Virtual Machine', - 'Rolling': 'Rolling Maintenance' + 'Rolling': 'Rolling Maintenance', + 'importVsphereStoragePolicies' : 'vSphere storage policies' } From a951e5f57fb2c8a867c8ace264d59d85e9011a5e Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Thu, 4 Jun 2020 15:37:16 +0530 Subject: [PATCH 009/164] Fixed unit test failure --- .../hypervisor/vmware/VmwareDatacenterApiUnitTest.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/vmware/VmwareDatacenterApiUnitTest.java b/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/vmware/VmwareDatacenterApiUnitTest.java index eb041396459..1249f6ce004 100644 --- a/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/vmware/VmwareDatacenterApiUnitTest.java +++ b/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/vmware/VmwareDatacenterApiUnitTest.java @@ -31,6 +31,7 @@ import com.cloud.dc.dao.ClusterDao; import com.cloud.dc.dao.ClusterVSMMapDao; import com.cloud.dc.dao.DataCenterDao; import com.cloud.dc.dao.HostPodDao; +import com.cloud.dc.dao.VsphereStoragePolicyDao; import com.cloud.event.dao.EventDao; import com.cloud.exception.DiscoveryException; import com.cloud.exception.InvalidParameterValueException; @@ -486,6 +487,11 @@ public class VmwareDatacenterApiUnitTest { return Mockito.mock(TemplateManager.class); } + @Bean + public VsphereStoragePolicyDao vsphereStoragePolicyDao() { + return Mockito.mock(VsphereStoragePolicyDao.class); + } + public static class Library implements TypeFilter { @Override From 27e5dfa633e2ddde2e89fa4cc2186332e1a4777e Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Fri, 5 Jun 2020 11:58:57 +0530 Subject: [PATCH 010/164] Updated code to handle sync of storage policies when importVsphereStoragePolicies API is called multiple times --- .../java/com/cloud/dc/VsphereStoragePolicyVO.java | 8 ++++++++ .../com/cloud/dc/dao/VsphereStoragePolicyDao.java | 3 +++ .../com/cloud/dc/dao/VsphereStoragePolicyDaoImpl.java | 9 +++++++++ .../hypervisor/vmware/manager/VmwareManagerImpl.java | 11 +++++++++-- 4 files changed, 29 insertions(+), 2 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/dc/VsphereStoragePolicyVO.java b/engine/schema/src/main/java/com/cloud/dc/VsphereStoragePolicyVO.java index 591455ed8a6..1415d15ec77 100644 --- a/engine/schema/src/main/java/com/cloud/dc/VsphereStoragePolicyVO.java +++ b/engine/schema/src/main/java/com/cloud/dc/VsphereStoragePolicyVO.java @@ -103,11 +103,19 @@ public class VsphereStoragePolicyVO implements VsphereStoragePolicy { return name; } + public void setName(String name) { + this.name = name; + } + @Override public String getDescription() { return description; } + public void setDescription(String description) { + this.description = description; + } + public Date getUpdateTime() { return updateTime; } diff --git a/engine/schema/src/main/java/com/cloud/dc/dao/VsphereStoragePolicyDao.java b/engine/schema/src/main/java/com/cloud/dc/dao/VsphereStoragePolicyDao.java index 907ab201818..e06fd36b6d4 100644 --- a/engine/schema/src/main/java/com/cloud/dc/dao/VsphereStoragePolicyDao.java +++ b/engine/schema/src/main/java/com/cloud/dc/dao/VsphereStoragePolicyDao.java @@ -20,4 +20,7 @@ import com.cloud.dc.VsphereStoragePolicyVO; import com.cloud.utils.db.GenericDao; public interface VsphereStoragePolicyDao extends GenericDao { + + public VsphereStoragePolicyVO findByPolicyId(Long zoneId, String policyId); + } diff --git a/engine/schema/src/main/java/com/cloud/dc/dao/VsphereStoragePolicyDaoImpl.java b/engine/schema/src/main/java/com/cloud/dc/dao/VsphereStoragePolicyDaoImpl.java index 4b2fd234132..fadea5cef25 100644 --- a/engine/schema/src/main/java/com/cloud/dc/dao/VsphereStoragePolicyDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/dc/dao/VsphereStoragePolicyDaoImpl.java @@ -39,7 +39,16 @@ public class VsphereStoragePolicyDaoImpl extends GenericDaoBase sc = policySearch.create(); + sc.setParameters("zoneId", zoneId); + sc.setParameters("policyId", policyId); + return findOneBy(sc); + } } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java index 85b7beeec69..9b900ad939c 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java @@ -1429,8 +1429,15 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw } for (PbmProfile storageProfile : storageProfiles) { - VsphereStoragePolicyVO StoragePolicyVO = new VsphereStoragePolicyVO(zoneId, storageProfile.getProfileId().getUniqueId(), storageProfile.getName(), storageProfile.getDescription()); - vsphereStoragePolicyDao.persist(StoragePolicyVO); + VsphereStoragePolicyVO storagePolicyVO = vsphereStoragePolicyDao.findByPolicyId(zoneId, storageProfile.getProfileId().getUniqueId()); + if (storagePolicyVO == null) { + storagePolicyVO = new VsphereStoragePolicyVO(zoneId, storageProfile.getProfileId().getUniqueId(), storageProfile.getName(), storageProfile.getDescription()); + vsphereStoragePolicyDao.persist(storagePolicyVO); + } else { + storagePolicyVO.setDescription(storageProfile.getDescription()); + storagePolicyVO.setName(storageProfile.getName()); + vsphereStoragePolicyDao.update(storagePolicyVO.getId(), storagePolicyVO); + } } List storagePolicies = vsphereStoragePolicyDao.listAll(); From fb0a96e7fb04d012355c2efad972fb802e5c874d Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Tue, 9 Jun 2020 12:56:13 +0530 Subject: [PATCH 011/164] Check if datastore is complaince with the storagepolicy provided in the disk offering. Added corresponding manager objects from PBM sdk to do the job. Made dao layer changes to read the storage policy in diskoffering --- .../storage/resource/StorageProcessor.java | 3 + .../StorageSubsystemCommandHandlerBase.java | 3 + ...taStoreStoragePolicyComplainceCommand.java | 61 +++++++++++++++++ .../com/cloud/storage/StorageManager.java | 2 + .../dao/DiskOfferingDetailsDao.java | 1 + .../dao/DiskOfferingDetailsDaoImpl.java | 10 +++ .../AbstractStoragePoolAllocator.java | 12 ++++ .../kvm/storage/KVMStorageProcessor.java | 7 ++ .../ovm3/resources/Ovm3StorageProcessor.java | 7 ++ .../resource/VmwareStorageProcessor.java | 33 +++++++++ .../resource/XenServerStorageProcessor.java | 7 ++ .../Xenserver625StorageProcessor.java | 7 ++ .../deploy/DeploymentPlanningManagerImpl.java | 13 ++++ .../com/cloud/storage/StorageManagerImpl.java | 54 ++++++++++++++- .../hypervisor/vmware/mo/DatacenterMO.java | 14 ++++ .../hypervisor/vmware/mo/DatastoreMO.java | 11 +++ .../vmware/mo/PbmPlacementSolverMO.java | 68 +++++++++++++++++++ .../vmware/mo/PbmProfileManagerMO.java | 15 ++++ 18 files changed, 327 insertions(+), 1 deletion(-) create mode 100644 core/src/main/java/org/apache/cloudstack/storage/command/CheckDataStoreStoragePolicyComplainceCommand.java create mode 100644 vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/PbmPlacementSolverMO.java diff --git a/core/src/main/java/com/cloud/storage/resource/StorageProcessor.java b/core/src/main/java/com/cloud/storage/resource/StorageProcessor.java index f940e22f45a..4b2438ea102 100644 --- a/core/src/main/java/com/cloud/storage/resource/StorageProcessor.java +++ b/core/src/main/java/com/cloud/storage/resource/StorageProcessor.java @@ -21,6 +21,7 @@ package com.cloud.storage.resource; import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand; import org.apache.cloudstack.storage.command.AttachCommand; +import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplainceCommand; import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.command.CreateObjectCommand; import org.apache.cloudstack.storage.command.DeleteCommand; @@ -76,4 +77,6 @@ public interface StorageProcessor { public Answer handleDownloadTemplateToPrimaryStorage(DirectDownloadCommand cmd); Answer copyVolumeFromPrimaryToPrimary(CopyCommand cmd); + + public Answer CheckDataStoreStoragePolicyComplaince(CheckDataStoreStoragePolicyComplainceCommand cmd); } diff --git a/core/src/main/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java b/core/src/main/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java index 17b9b700d6c..910eb3d8790 100644 --- a/core/src/main/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java +++ b/core/src/main/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java @@ -21,6 +21,7 @@ package com.cloud.storage.resource; import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand; import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplainceCommand; import org.apache.log4j.Logger; import org.apache.cloudstack.storage.command.AttachCommand; @@ -71,6 +72,8 @@ public class StorageSubsystemCommandHandlerBase implements StorageSubsystemComma return processor.resignature((ResignatureCommand) command); } else if (command instanceof DirectDownloadCommand) { return processor.handleDownloadTemplateToPrimaryStorage((DirectDownloadCommand) command); + } else if (command instanceof CheckDataStoreStoragePolicyComplainceCommand) { + return processor.CheckDataStoreStoragePolicyComplaince((CheckDataStoreStoragePolicyComplainceCommand) command); } return new Answer((Command)command, false, "not implemented yet"); diff --git a/core/src/main/java/org/apache/cloudstack/storage/command/CheckDataStoreStoragePolicyComplainceCommand.java b/core/src/main/java/org/apache/cloudstack/storage/command/CheckDataStoreStoragePolicyComplainceCommand.java new file mode 100644 index 00000000000..f9544b873ef --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/storage/command/CheckDataStoreStoragePolicyComplainceCommand.java @@ -0,0 +1,61 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.storage.command; + +import com.cloud.agent.api.to.StorageFilerTO; + +public class CheckDataStoreStoragePolicyComplainceCommand extends StorageSubSystemCommand { + + String storagePolicyId; + private StorageFilerTO storagePool; + + public CheckDataStoreStoragePolicyComplainceCommand(String storagePolicyId, StorageFilerTO storagePool) { + super(); + + this.storagePolicyId = storagePolicyId; + this.storagePool = storagePool; + } + + @Override + public void setExecuteInSequence(boolean inSeq) { + } + + @Override + public boolean executeInSequence() { + return false; + } + + + public String getStoragePolicyId() { + return storagePolicyId; + } + + public void setStoragePolicyId(String storagePolicyId) { + this.storagePolicyId = storagePolicyId; + } + + public StorageFilerTO getStoragePool() { + return storagePool; + } + + public void setStoragePool(StorageFilerTO storagePool) { + this.storagePool = storagePool; + } +} diff --git a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java index 0f52206dd78..b63574cd808 100644 --- a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java +++ b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java @@ -212,6 +212,8 @@ public interface StorageManager extends StorageService { boolean storagePoolHasEnoughSpaceForResize(StoragePool pool, long currentSize, long newSiz); + boolean isStoragePoolComplaintWithStoragePolicy(List volumes, StoragePool pool) throws StorageUnavailableException; + boolean registerHostListener(String providerUuid, HypervisorHostListener listener); void connectHostToSharedPool(long hostId, long poolId) throws StorageUnavailableException, StorageConflictException; diff --git a/engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/dao/DiskOfferingDetailsDao.java b/engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/dao/DiskOfferingDetailsDao.java index e201ae27fdc..815f1693ee5 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/dao/DiskOfferingDetailsDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/dao/DiskOfferingDetailsDao.java @@ -26,4 +26,5 @@ import com.cloud.utils.db.GenericDao; public interface DiskOfferingDetailsDao extends GenericDao, ResourceDetailsDao { List findDomainIds(final long resourceId); List findZoneIds(final long resourceId); + String getDetail(Long diskOfferingId, String key); } \ No newline at end of file diff --git a/engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/dao/DiskOfferingDetailsDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/dao/DiskOfferingDetailsDaoImpl.java index da0ec5bc580..5408f2d7f03 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/dao/DiskOfferingDetailsDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/dao/DiskOfferingDetailsDaoImpl.java @@ -56,4 +56,14 @@ public class DiskOfferingDetailsDaoImpl extends ResourceDetailsDaoBase requestVolumes = new ArrayList<>(); requestVolumes.add(volume); + if (dskCh.getHypervisorType() == HypervisorType.VMware) { + try { + boolean isStoragePoolStoragepolicyComplaince = storageMgr.isStoragePoolComplaintWithStoragePolicy(requestVolumes, pool); + if (!isStoragePoolStoragepolicyComplaince) { + return false; + } + } catch (StorageUnavailableException e) { + s_logger.warn(String.format("Could not verify storage policy complaince against storage pool %s due to exception %s", pool.getUuid(), e.getMessage())); + return false; + } + } return storageMgr.storagePoolHasEnoughIops(requestVolumes, pool) && storageMgr.storagePoolHasEnoughSpace(requestVolumes, pool, plan.getClusterId()); } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index b792ff22204..cc47c553283 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -46,6 +46,7 @@ import org.apache.cloudstack.agent.directdownload.MetalinkDirectDownloadCommand; import org.apache.cloudstack.agent.directdownload.NfsDirectDownloadCommand; import org.apache.cloudstack.storage.command.AttachAnswer; import org.apache.cloudstack.storage.command.AttachCommand; +import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplainceCommand; import org.apache.cloudstack.storage.command.CopyCmdAnswer; import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.command.CreateObjectAnswer; @@ -1833,4 +1834,10 @@ public class KVMStorageProcessor implements StorageProcessor { } return availableBytes >= templateSize; } + + @Override + public Answer CheckDataStoreStoragePolicyComplaince(CheckDataStoreStoragePolicyComplainceCommand cmd) { + s_logger.info("'CheckDataStoreStoragePolicyComplainceCommand' not currently applicable for KVMStorageProcessor"); + return new Answer(cmd,false,"Not currently applicable for KVMStorageProcessor"); + } } diff --git a/plugins/hypervisors/ovm3/src/main/java/com/cloud/hypervisor/ovm3/resources/Ovm3StorageProcessor.java b/plugins/hypervisors/ovm3/src/main/java/com/cloud/hypervisor/ovm3/resources/Ovm3StorageProcessor.java index 7915586fca3..dd58bb573d9 100644 --- a/plugins/hypervisors/ovm3/src/main/java/com/cloud/hypervisor/ovm3/resources/Ovm3StorageProcessor.java +++ b/plugins/hypervisors/ovm3/src/main/java/com/cloud/hypervisor/ovm3/resources/Ovm3StorageProcessor.java @@ -24,6 +24,7 @@ import java.util.UUID; import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand; import org.apache.cloudstack.storage.command.AttachAnswer; import org.apache.cloudstack.storage.command.AttachCommand; +import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplainceCommand; import org.apache.cloudstack.storage.command.CopyCmdAnswer; import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.command.CreateObjectAnswer; @@ -826,6 +827,12 @@ public class Ovm3StorageProcessor implements StorageProcessor { return null; } + @Override + public Answer CheckDataStoreStoragePolicyComplaince(CheckDataStoreStoragePolicyComplainceCommand cmd) { + LOGGER.info("'CheckDataStoreStoragePolicyComplainceCommand' not applicable used for Ovm3StorageProcessor"); + return new Answer(cmd,false,"Not applicable used for Ovm3StorageProcessor"); + } + @Override public Answer copyVolumeFromPrimaryToPrimary(CopyCommand cmd) { return null; diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java index 5c9c91767bd..1e30101933d 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -37,6 +37,7 @@ import java.util.concurrent.TimeUnit; import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand; import org.apache.cloudstack.storage.command.AttachAnswer; import org.apache.cloudstack.storage.command.AttachCommand; +import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplainceCommand; import org.apache.cloudstack.storage.command.CopyCmdAnswer; import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.command.CreateObjectAnswer; @@ -3580,6 +3581,38 @@ public class VmwareStorageProcessor implements StorageProcessor { return null; } + @Override + public Answer CheckDataStoreStoragePolicyComplaince(CheckDataStoreStoragePolicyComplainceCommand cmd) { + String primaryStorageNameLabel = cmd.getStoragePool().getUuid(); + String storagePolicyId = cmd.getStoragePolicyId(); + VmwareContext context = hostService.getServiceContext(cmd); + try { + VmwareHypervisorHost hyperHost = hostService.getHyperHost(context, cmd); + ManagedObjectReference morPrimaryDs = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, primaryStorageNameLabel); + if (morPrimaryDs == null) { + String msg = "Unable to find datastore: " + primaryStorageNameLabel; + s_logger.error(msg); + throw new Exception(msg); + } + + DatastoreMO primaryDsMo = new DatastoreMO(hyperHost.getContext(), morPrimaryDs); + boolean isDatastoreStoragePolicyComplaint = primaryDsMo.isDatastoreStoragePolicyComplaint(storagePolicyId); + + String failedMessage = String.format("DataStore %s is not complaince with storage policy id %s", primaryStorageNameLabel, storagePolicyId); + if (!isDatastoreStoragePolicyComplaint) + return new Answer(cmd, isDatastoreStoragePolicyComplaint, failedMessage); + else + return new Answer(cmd, isDatastoreStoragePolicyComplaint, null); + } catch (Throwable e) { + if (e instanceof RemoteException) { + hostService.invalidateServiceContext(context); + } + String details = String.format("Exception while checking if datastore %s is storage policy %s complaince : %s", primaryStorageNameLabel, storagePolicyId, VmwareHelper.getExceptionMessage(e)); + s_logger.error(details, e); + return new Answer(cmd, false, details); + } + } + @Override public Answer copyVolumeFromPrimaryToPrimary(CopyCommand cmd) { return null; diff --git a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java index d987f28ca94..d530969941b 100644 --- a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java +++ b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java @@ -35,6 +35,7 @@ import java.util.UUID; import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand; import org.apache.cloudstack.storage.command.AttachAnswer; import org.apache.cloudstack.storage.command.AttachCommand; +import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplainceCommand; import org.apache.cloudstack.storage.command.CopyCmdAnswer; import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.command.CreateObjectAnswer; @@ -215,6 +216,12 @@ public class XenServerStorageProcessor implements StorageProcessor { return null; } + @Override + public Answer CheckDataStoreStoragePolicyComplaince(CheckDataStoreStoragePolicyComplainceCommand cmd) { + s_logger.info("'CheckDataStoreStoragePolicyComplainceCommand' not applicable used for XenServerStorageProcessor"); + return new Answer(cmd,false,"Not applicable used for XenServerStorageProcessor"); + } + @Override public AttachAnswer attachIso(final AttachCommand cmd) { final DiskTO disk = cmd.getDisk(); diff --git a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/Xenserver625StorageProcessor.java b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/Xenserver625StorageProcessor.java index 184013e42a4..9b54e0bc1f7 100644 --- a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/Xenserver625StorageProcessor.java +++ b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/Xenserver625StorageProcessor.java @@ -27,6 +27,7 @@ import java.util.Map; import java.util.Set; import java.util.UUID; +import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplainceCommand; import org.apache.cloudstack.storage.command.CopyCmdAnswer; import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; @@ -912,6 +913,12 @@ public class Xenserver625StorageProcessor extends XenServerStorageProcessor { return new CopyCmdAnswer(details); } + @Override + public Answer CheckDataStoreStoragePolicyComplaince(CheckDataStoreStoragePolicyComplainceCommand cmd) { + s_logger.info("'CheckDataStoreStoragePolicyComplainceCommand' not applicable used for XenServerStorageProcessor"); + return new Answer(cmd,false,"Not applicable used for XenServerStorageProcessor"); + } + @Override public Answer copyVolumeFromPrimaryToSecondary(final CopyCommand cmd) { final Connection conn = hypervisorResource.getConnection(); diff --git a/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java b/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java index 2149a5681d0..340d9119de8 100644 --- a/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java +++ b/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java @@ -31,6 +31,7 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; import com.cloud.utils.StringUtils; +import com.cloud.exception.StorageUnavailableException; import com.cloud.utils.db.Filter; import com.cloud.utils.fsm.StateMachine2; @@ -1291,6 +1292,18 @@ StateListener { requestVolumes = new ArrayList(); requestVolumes.add(vol); + if (potentialHost.getHypervisorType() == HypervisorType.VMware) { + try { + boolean isStoragePoolStoragepolicyComplaince = _storageMgr.isStoragePoolComplaintWithStoragePolicy(requestVolumes, potentialSPool); + if (!isStoragePoolStoragepolicyComplaince) { + continue; + } + } catch (StorageUnavailableException e) { + s_logger.warn(String.format("Could not verify storage policy complaince against storage pool %s due to exception %s", potentialSPool.getUuid(), e.getMessage())); + continue; + } + } + if (!_storageMgr.storagePoolHasEnoughIops(requestVolumes, potentialSPool) || !_storageMgr.storagePoolHasEnoughSpace(requestVolumes, potentialSPool, potentialHost.getClusterId())) continue; diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index c59a26d3bc6..db2ebd72d79 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -39,6 +39,11 @@ import java.util.concurrent.TimeUnit; import javax.inject.Inject; +import com.cloud.agent.api.to.StorageFilerTO; +import com.cloud.dc.VsphereStoragePolicyVO; +import com.cloud.dc.dao.VsphereStoragePolicyDao; +import com.cloud.utils.StringUtils; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.command.admin.storage.CancelPrimaryStorageMaintenanceCmd; import org.apache.cloudstack.api.command.admin.storage.CreateSecondaryStagingStoreCmd; import org.apache.cloudstack.api.command.admin.storage.CreateStoragePoolCmd; @@ -81,6 +86,8 @@ import org.apache.cloudstack.framework.config.Configurable; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.cloudstack.management.ManagementServerHost; +import org.apache.cloudstack.resourcedetail.dao.DiskOfferingDetailsDao; +import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplainceCommand; import org.apache.cloudstack.storage.command.DettachCommand; import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; import org.apache.cloudstack.storage.datastore.db.ImageStoreDetailsDao; @@ -175,7 +182,6 @@ import com.cloud.user.dao.UserDao; import com.cloud.utils.DateUtil; import com.cloud.utils.NumbersUtil; import com.cloud.utils.Pair; -import com.cloud.utils.StringUtils; import com.cloud.utils.UriUtils; import com.cloud.utils.component.ComponentContext; import com.cloud.utils.component.ManagerBase; @@ -298,6 +304,10 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C SnapshotService _snapshotService; @Inject StoragePoolTagsDao _storagePoolTagsDao; + @Inject + DiskOfferingDetailsDao _diskOfferingDetailsDao; + @Inject + VsphereStoragePolicyDao _vsphereStoragePolicyDao; protected List _discoverers; @@ -1872,6 +1882,48 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C } } + @Override + public boolean isStoragePoolComplaintWithStoragePolicy(List volumes, StoragePool pool) throws StorageUnavailableException { + if (volumes == null || volumes.isEmpty()) { + return false; + } + List> answers = new ArrayList>(); + + for (Volume volume : volumes) { + String storagePolicyId = _diskOfferingDetailsDao.getDetail(volume.getDiskOfferingId(), ApiConstants.STORAGE_POLICY); + if (org.apache.commons.lang.StringUtils.isNotEmpty(storagePolicyId)) { + VsphereStoragePolicyVO storagePolicyVO = _vsphereStoragePolicyDao.findById(Long.parseLong(storagePolicyId)); + List hostIds = getUpHostsInPool(pool.getId()); + Collections.shuffle(hostIds); + + if (hostIds == null || hostIds.isEmpty()) { + throw new StorageUnavailableException("Unable to send command to the pool " + pool.getName() + " due to there is no enabled hosts up in this cluster", pool.getId()); + } + try { + StorageFilerTO storageFilerTO = new StorageFilerTO(pool); + CheckDataStoreStoragePolicyComplainceCommand cmd = new CheckDataStoreStoragePolicyComplainceCommand(storagePolicyVO.getPolicyId(), storageFilerTO); + long targetHostId = _hvGuruMgr.getGuruProcessedCommandTargetHost(hostIds.get(0), cmd); + Answer answer = _agentMgr.send(targetHostId, cmd); + answers.add(new Pair<>(volume, answer)); + } catch (AgentUnavailableException e) { + s_logger.debug("Unable to send storage pool command to " + pool + " via " + hostIds.get(0), e); + throw new StorageUnavailableException("Unable to send command to the pool ", pool.getId()); + } catch (OperationTimedoutException e) { + s_logger.debug("Failed to process storage pool command to " + pool + " via " + hostIds.get(0), e); + throw new StorageUnavailableException("Failed to process storage command to the pool ", pool.getId()); + } + } + } + // check cummilative result for all volumes + for (Pair answer : answers) { + if (!answer.second().getResult()) { + s_logger.debug(String.format("Storage pool %s is not complaince with storage policy for volume %s", pool.getName(), answer.first().getName())); + return false; + } + } + return true; + } + private boolean checkPoolforSpace(StoragePool pool, long allocatedSizeWithTemplate, long totalAskingSize) { // allocated space includes templates StoragePoolVO poolVO = _storagePoolDao.findById(pool.getId()); diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatacenterMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatacenterMO.java index b0b91fb7d5b..af89757d521 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatacenterMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatacenterMO.java @@ -186,6 +186,20 @@ public class DatacenterMO extends BaseMO { return null; } + public ManagedObjectReference listDatastore(String name) throws Exception { + assert (name != null); + + List ocs = getDatastorePropertiesOnDatacenter(new String[] {"name"}); + if (ocs != null) { + for (ObjectContent oc : ocs) { + if (oc.getPropSet().get(0).getVal().toString().equals(name)) { + return oc.getObj(); + } + } + } + return null; + } + public ManagedObjectReference findHost(String name) throws Exception { List ocs = getHostPropertiesOnDatacenterHostFolder(new String[] {"name"}); diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatastoreMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatastoreMO.java index 19ae41f6c77..3e914a4c277 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatastoreMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatastoreMO.java @@ -19,6 +19,7 @@ package com.cloud.hypervisor.vmware.mo; import java.util.ArrayList; import java.util.List; +import com.vmware.pbm.PbmProfile; import org.apache.log4j.Logger; import com.vmware.vim25.DatastoreHostMount; @@ -471,4 +472,14 @@ public class DatastoreMO extends BaseMO { } return isAccessible; } + + public boolean isDatastoreStoragePolicyComplaint(String storagePolicyId) throws Exception { + PbmProfileManagerMO profMgrMo = new PbmProfileManagerMO(_context); + PbmProfile profile = profMgrMo.getStorageProfile(storagePolicyId); + + PbmPlacementSolverMO placementSolverMo = new PbmPlacementSolverMO(_context); + boolean isDatastoreCompatible = placementSolverMo.isDatastoreCompatibleWithStorageProfile(_mor, profile); + + return isDatastoreCompatible; + } } diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/PbmPlacementSolverMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/PbmPlacementSolverMO.java new file mode 100644 index 00000000000..9735f424d88 --- /dev/null +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/PbmPlacementSolverMO.java @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.hypervisor.vmware.mo; + +import com.cloud.hypervisor.vmware.util.VmwareContext; +import com.vmware.pbm.PbmPlacementCompatibilityResult; +import com.vmware.pbm.PbmPlacementHub; +import com.vmware.pbm.PbmProfile; +import com.vmware.pbm.PbmProfileId; +import com.vmware.vim25.ManagedObjectReference; +import org.apache.commons.collections.CollectionUtils; +import org.apache.log4j.Logger; + +import java.util.ArrayList; +import java.util.List; + +public class PbmPlacementSolverMO extends BaseMO { + + private static final Logger s_logger = Logger.getLogger(PbmPlacementSolverMO.class); + + public PbmPlacementSolverMO (VmwareContext context) { + super(context, context.getPbmServiceContent().getPlacementSolver()); + } + + public PbmPlacementSolverMO(VmwareContext context, ManagedObjectReference morPlacementSolver) { + super(context, morPlacementSolver); + } + + public PbmPlacementSolverMO(VmwareContext context, String morType, String morValue) { + super(context, morType, morValue); + } + + public boolean isDatastoreCompatibleWithStorageProfile(ManagedObjectReference dsMor, PbmProfile profile) throws Exception { + boolean isDatastoreCompatibleWithStorageProfile = false; + + PbmPlacementHub placementHub = new PbmPlacementHub(); + placementHub.setHubId(dsMor.getValue()); + placementHub.setHubType(dsMor.getType()); + + List placementHubList = new ArrayList(); + placementHubList.add(placementHub); + PbmProfileId profileId = profile.getProfileId(); + List placementCompatibilityResultList = _context.getPbmService().pbmCheckCompatibility(_mor, placementHubList, profileId); + if (CollectionUtils.isNotEmpty(placementCompatibilityResultList)) { + for (PbmPlacementCompatibilityResult placementResult : placementCompatibilityResultList) { + // Check for error and warning + if (CollectionUtils.isEmpty(placementResult.getError()) && CollectionUtils.isEmpty(placementResult.getWarning())) { + isDatastoreCompatibleWithStorageProfile = true; + } + } + } + return isDatastoreCompatibleWithStorageProfile; + } +} diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/PbmProfileManagerMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/PbmProfileManagerMO.java index 9109b18f7b3..557f350cf8e 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/PbmProfileManagerMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/PbmProfileManagerMO.java @@ -18,6 +18,7 @@ package com.cloud.hypervisor.vmware.mo; import com.cloud.hypervisor.vmware.util.VmwareContext; +import com.cloud.utils.exception.CloudRuntimeException; import com.vmware.pbm.PbmProfile; import com.vmware.pbm.PbmProfileId; import com.vmware.pbm.PbmProfileResourceType; @@ -26,6 +27,7 @@ import com.vmware.vim25.ManagedObjectReference; import org.apache.log4j.Logger; +import java.util.Collections; import java.util.List; public class PbmProfileManagerMO extends BaseMO { @@ -58,6 +60,19 @@ public class PbmProfileManagerMO extends BaseMO { return profiles; } + public PbmProfile getStorageProfile(String storageProfileId) throws Exception { + List profileIds = getStorageProfileIds(); + for (PbmProfileId profileId : profileIds) { + if (storageProfileId.equals(profileId.getUniqueId())) { + List profile = _context.getPbmService().pbmRetrieveContent(_mor, Collections.singletonList(profileId)); + return profile.get(0); + } + } + String errMsg = String.format("Storage profile with id %s not found", storageProfileId); + s_logger.debug(errMsg); + throw new CloudRuntimeException(errMsg); + } + private PbmProfileResourceType getStorageResourceType() { PbmProfileResourceType resourceType = new PbmProfileResourceType(); resourceType.setResourceType(PbmProfileResourceTypeEnum.STORAGE.value()); From 487bb03c4d6d23faa7142c5c8636d2b0839880cc Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Fri, 5 Jun 2020 14:58:27 +0530 Subject: [PATCH 012/164] Added Storagepolicy while creating diskoffering. CreateDiskOffering API now takes storagepolicy as a parameter with UUID value --- .../admin/offering/CreateDiskOfferingCmd.java | 7 +++++++ .../ImportVsphereStoragePoliciesResponse.java | 0 .../ConfigurationManagerImpl.java | 20 ++++++++++++++++--- 3 files changed, 24 insertions(+), 3 deletions(-) rename {plugins/hypervisors/vmware => api}/src/main/java/org/apache/cloudstack/api/response/ImportVsphereStoragePoliciesResponse.java (100%) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateDiskOfferingCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateDiskOfferingCmd.java index f0ca5fb851a..4f9eebeb253 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateDiskOfferingCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateDiskOfferingCmd.java @@ -28,6 +28,7 @@ import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.response.DiskOfferingResponse; import org.apache.cloudstack.api.response.DomainResponse; +import org.apache.cloudstack.api.response.ImportVsphereStoragePoliciesResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; @@ -151,6 +152,9 @@ public class CreateDiskOfferingCmd extends BaseCmd { since = "4.14") private String cacheMode; + @Parameter(name = ApiConstants.STORAGE_POLICY, type = CommandType.UUID, entityType = ImportVsphereStoragePoliciesResponse.class,required = false, description = "Name of the storage policy defined at vCenter, this is applicable only for VMware") + private Long storagePolicy; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -273,6 +277,9 @@ public class CreateDiskOfferingCmd extends BaseCmd { return cacheMode; } + public Long getStoragePolicy() { + return storagePolicy; + } ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/response/ImportVsphereStoragePoliciesResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/ImportVsphereStoragePoliciesResponse.java similarity index 100% rename from plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/response/ImportVsphereStoragePoliciesResponse.java rename to api/src/main/java/org/apache/cloudstack/api/response/ImportVsphereStoragePoliciesResponse.java diff --git a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java index ac2ff0762bc..e8c0284b14f 100755 --- a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java +++ b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java @@ -39,7 +39,7 @@ import java.util.UUID; import javax.inject.Inject; import javax.naming.ConfigurationException; -import com.cloud.utils.StringUtils; +import com.cloud.dc.dao.VsphereStoragePolicyDao; import org.apache.cloudstack.acl.SecurityChecker; import org.apache.cloudstack.affinity.AffinityGroup; import org.apache.cloudstack.affinity.AffinityGroupService; @@ -223,6 +223,7 @@ import com.cloud.user.dao.AccountDao; import com.cloud.user.dao.UserDao; import com.cloud.utils.NumbersUtil; import com.cloud.utils.Pair; +import com.cloud.utils.StringUtils; import com.cloud.utils.UriUtils; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.db.DB; @@ -387,6 +388,9 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati IndirectAgentLB _indirectAgentLB; @Inject private VMTemplateZoneDao templateZoneDao; + @Inject + VsphereStoragePolicyDao vsphereStoragePolicyDao; + // FIXME - why don't we have interface for DataCenterLinkLocalIpAddressDao? @Inject @@ -2827,7 +2831,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati Long bytesWriteRate, Long bytesWriteRateMax, Long bytesWriteRateMaxLength, Long iopsReadRate, Long iopsReadRateMax, Long iopsReadRateMaxLength, Long iopsWriteRate, Long iopsWriteRateMax, Long iopsWriteRateMaxLength, - final Integer hypervisorSnapshotReserve, String cacheMode) { + final Integer hypervisorSnapshotReserve, String cacheMode, final Long storagePolicyID) { long diskSize = 0;// special case for custom disk offerings if (numGibibytes != null && numGibibytes <= 0) { throw new InvalidParameterValueException("Please specify a disk size of at least 1 Gb."); @@ -2955,6 +2959,9 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati detailsVO.add(new DiskOfferingDetailVO(offering.getId(), ApiConstants.ZONE_ID, String.valueOf(zoneId), false)); } } + if (storagePolicyID != null) { + detailsVO.add(new DiskOfferingDetailVO(offering.getId(), ApiConstants.STORAGE_POLICY, String.valueOf(storagePolicyID), false)); + } if (!detailsVO.isEmpty()) { diskOfferingDetailsDao.saveDetails(detailsVO); } @@ -2976,6 +2983,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati final String tags = cmd.getTags(); final List domainIds = cmd.getDomainIds(); final List zoneIds = cmd.getZoneIds(); + final Long storagePolicyId = cmd.getStoragePolicy(); // check if valid domain if (CollectionUtils.isNotEmpty(domainIds)) { @@ -3015,6 +3023,12 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati } } + if (storagePolicyId != null) { + if (vsphereStoragePolicyDao.findById(storagePolicyId) == null) { + throw new InvalidParameterValueException("Please specify a valid vSphere storage policy id"); + } + } + final Boolean isCustomizedIops = cmd.isCustomizedIops(); final Long minIops = cmd.getMinIops(); final Long maxIops = cmd.getMaxIops(); @@ -3045,7 +3059,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati localStorageRequired, isDisplayOfferingEnabled, isCustomizedIops, minIops, maxIops, bytesReadRate, bytesReadRateMax, bytesReadRateMaxLength, bytesWriteRate, bytesWriteRateMax, bytesWriteRateMaxLength, iopsReadRate, iopsReadRateMax, iopsReadRateMaxLength, iopsWriteRate, iopsWriteRateMax, iopsWriteRateMaxLength, - hypervisorSnapshotReserve, cacheMode); + hypervisorSnapshotReserve, cacheMode, storagePolicyId); } /** From 851b562e042575b5b39a2fdfe820bf3aea43d45f Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Mon, 15 Jun 2020 08:44:26 +0530 Subject: [PATCH 013/164] Unit test fix --- server/src/test/resources/createNetworkOffering.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/test/resources/createNetworkOffering.xml b/server/src/test/resources/createNetworkOffering.xml index 8dee0e8a54e..55343ef835d 100644 --- a/server/src/test/resources/createNetworkOffering.xml +++ b/server/src/test/resources/createNetworkOffering.xml @@ -60,4 +60,5 @@ + From 6c314492e756b945c928b8f89104ef564a4b7a3e Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Tue, 16 Jun 2020 12:07:53 +0530 Subject: [PATCH 014/164] Fix NPE in case of preparing secondarystorage on Host --- .../src/main/java/com/cloud/hypervisor/vmware/mo/HostMO.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HostMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HostMO.java index 100f56c9441..8c02fb444f3 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HostMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HostMO.java @@ -886,7 +886,9 @@ public class HostMO extends BaseMO implements VmwareHypervisorHost { } } - HypervisorHostHelper.createBaseFolderInDatastore(dsMo, this); + if (dsMo != null) { + HypervisorHostHelper.createBaseFolderInDatastore(dsMo, this); + } if (s_logger.isTraceEnabled()) s_logger.trace("vCenter API trace - mountDatastore() done(successfully)"); From aa07959f2adc77a471582a995af2aeb242d10dc0 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Fri, 26 Jun 2020 12:02:58 +0530 Subject: [PATCH 015/164] Use VStorageObjectManager for disk operations. Created disks using VStorageObjectManager Removed redundant code around attach volume and create volumes --- .../resource/VmwareStorageLayoutHelper.java | 9 +- .../resource/VmwareStorageProcessor.java | 69 +++++--- .../vmware/mo/HypervisorHostHelper.java | 1 + .../cloud/hypervisor/vmware/mo/TaskMO.java | 4 + .../vmware/mo/VirtualMachineMO.java | 105 ++++-------- .../mo/VirtualStorageObjectManager.java | 58 ------- .../mo/VirtualStorageObjectManagerMO.java | 94 +++++++++++ .../hypervisor/vmware/util/VmwareHelper.java | 150 +++++------------- 8 files changed, 214 insertions(+), 276 deletions(-) delete mode 100644 vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualStorageObjectManager.java create mode 100644 vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualStorageObjectManagerMO.java diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageLayoutHelper.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageLayoutHelper.java index a422fedb25f..91970e9073b 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageLayoutHelper.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageLayoutHelper.java @@ -311,7 +311,14 @@ public class VmwareStorageLayoutHelper { } public static String getLegacyDatastorePathFromVmdkFileName(DatastoreMO dsMo, String vmdkFileName) throws Exception { - return String.format("[%s] %s/%s", dsMo.getName(), HypervisorHostHelper.VSPHERE_DATASTORE_BASE_FOLDER, vmdkFileName); + String vmdkDatastorePath = String.format("[%s] %s/%s", dsMo.getName(), HypervisorHostHelper.VSPHERE_DATASTORE_BASE_FOLDER, vmdkFileName); + if (!dsMo.fileExists(vmdkDatastorePath)) { + vmdkDatastorePath = String.format("[%s] %s/%s", dsMo.getName(), HypervisorHostHelper.VSPHERE_FCD_DEFAULT_FOLDER, vmdkFileName); + } + if (!dsMo.fileExists(vmdkDatastorePath)) { + vmdkDatastorePath = getDeprecatedLegacyDatastorePathFromVmdkFileName(dsMo, vmdkFileName); + } + return vmdkDatastorePath; } public static String getDeprecatedLegacyDatastorePathFromVmdkFileName(DatastoreMO dsMo, String vmdkFileName) throws Exception { diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java index 1e30101933d..0e12d6e1d75 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -34,6 +34,10 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import com.cloud.hypervisor.vmware.mo.VirtualStorageObjectManagerMO; +import com.vmware.vim25.BaseConfigInfoDiskFileBackingInfo; +import com.vmware.vim25.VStorageObject; +import com.vmware.vim25.VirtualDiskType; import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand; import org.apache.cloudstack.storage.command.AttachAnswer; import org.apache.cloudstack.storage.command.AttachCommand; @@ -2253,37 +2257,50 @@ public class VmwareStorageProcessor implements StorageProcessor { String volumeUuid = UUID.randomUUID().toString().replace("-", ""); String volumeDatastorePath = dsMo.getDatastorePath(volumeUuid + ".vmdk"); - String dummyVmName = hostService.getWorkerName(context, cmd, 0); - try { - s_logger.info("Create worker VM " + dummyVmName); - vmMo = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, dummyVmName, null); - if (vmMo == null) { - throw new Exception("Unable to create a dummy VM for volume creation"); - } - synchronized (this) { - try { - vmMo.createDisk(volumeDatastorePath, (int)(volume.getSize() / (1024L * 1024L)), morDatastore, vmMo.getScsiDeviceControllerKey()); - vmMo.detachDisk(volumeDatastorePath, false); + VirtualStorageObjectManagerMO vStorageObjectManagerMO = new VirtualStorageObjectManagerMO(context); + VStorageObject virtualDisk = vStorageObjectManagerMO.createDisk(morDatastore, VirtualDiskType.THIN, volume.getSize(), volumeDatastorePath, volumeUuid); + VolumeObjectTO newVol = new VolumeObjectTO(); + DatastoreFile file = new DatastoreFile(((BaseConfigInfoDiskFileBackingInfo)virtualDisk.getConfig().getBacking()).getFilePath()); + newVol.setPath(file.getFileBaseName()); + newVol.setSize(volume.getSize()); + return new CreateObjectAnswer(newVol); + + /* + * // This is old code which uses workervm to create disks + * String dummyVmName = hostService.getWorkerName(context, cmd, 0); + try { + s_logger.info("Create worker VM " + dummyVmName); + vmMo = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, dummyVmName, null); + if (vmMo == null) { + throw new Exception("Unable to create a dummy VM for volume creation"); } - catch (Exception e) { - s_logger.error("Deleting file " + volumeDatastorePath + " due to error: " + e.getMessage()); - VmwareStorageLayoutHelper.deleteVolumeVmdkFiles(dsMo, volumeUuid, dcMo, VmwareManager.s_vmwareSearchExcludeFolder.value()); - throw new CloudRuntimeException("Unable to create volume due to: " + e.getMessage()); + + synchronized (this) { + try { + vmMo.createDisk(volumeDatastorePath, (int)(volume.getSize() / (1024L * 1024L)), morDatastore, vmMo.getScsiDeviceControllerKey()); + vmMo.detachDisk(volumeDatastorePath, false); + } + catch (Exception e) { + s_logger.error("Deleting file " + volumeDatastorePath + " due to error: " + e.getMessage()); + VmwareStorageLayoutHelper.deleteVolumeVmdkFiles(dsMo, volumeUuid, dcMo, VmwareManager.s_vmwareSearchExcludeFolder.value()); + throw new CloudRuntimeException("Unable to create volume due to: " + e.getMessage()); + } + } + + VolumeObjectTO newVol = new VolumeObjectTO(); + newVol.setPath(volumeUuid); + newVol.setSize(volume.getSize()); + return new CreateObjectAnswer(newVol); + } finally { + s_logger.info("Destroy dummy VM after volume creation"); + if (vmMo != null) { + vmMo.detachAllDisks(); + vmMo.destroy(); } } - VolumeObjectTO newVol = new VolumeObjectTO(); - newVol.setPath(volumeUuid); - newVol.setSize(volume.getSize()); - return new CreateObjectAnswer(newVol); - } finally { - s_logger.info("Destroy dummy VM after volume creation"); - if (vmMo != null) { - vmMo.detachAllDisks(); - vmMo.destroy(); - } - } + * */ } catch (Throwable e) { if (e instanceof RemoteException) { s_logger.warn("Encounter remote exception to vCenter, invalidate VMware session context"); diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java index 21e4f4c9e7c..6fd1a31c865 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java @@ -139,6 +139,7 @@ public class HypervisorHostHelper { private static final String VMDK_PACK_DIR = "ova"; private static final String OVA_OPTION_KEY_BOOTDISK = "cloud.ova.bootdisk"; public static final String VSPHERE_DATASTORE_BASE_FOLDER = ".cloudstack.base.folder"; + public static final String VSPHERE_FCD_DEFAULT_FOLDER = "fcd"; public static VirtualMachineMO findVmFromObjectContent(VmwareContext context, ObjectContent[] ocs, String name, String instanceNameCustomField) { diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/TaskMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/TaskMO.java index 65c6a6bd362..9cf9d9554c4 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/TaskMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/TaskMO.java @@ -77,4 +77,8 @@ public class TaskMO extends BaseMO { return sb.toString(); } + + public static TaskInfo getTaskInfo(VmwareContext context, ManagedObjectReference morTask) throws Exception { + return (TaskInfo)context.getVimClient().getDynamicProperty(morTask, "info"); + } } diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java index e9c8aafcbd1..3100b7f26ff 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java @@ -38,6 +38,7 @@ import com.vmware.vim25.VStorageObject; import com.vmware.vim25.VStorageObjectConfigInfo; import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; +import org.apache.commons.lang.StringUtils; import com.google.gson.Gson; import com.vmware.vim25.ArrayOfManagedObjectReference; @@ -1195,7 +1196,18 @@ public class VirtualMachineMO extends BaseMO { s_logger.trace("vCenter API trace - createDisk() done(successfully)"); } - public void updateVmdkAdapter(String vmdkFileName, String newAdapterType) throws Exception { + public void updateVmdkAdapter(String vmdkFileName, String diskController) throws Exception { + + DiskControllerType diskControllerType = DiskControllerType.getType(diskController); + VmdkAdapterType vmdkAdapterType = VmdkAdapterType.getAdapterType(diskControllerType); + if (vmdkAdapterType == VmdkAdapterType.none) { + String message = "Failed to attach disk due to invalid vmdk adapter type for vmdk file [" + + vmdkFileName + "] with controller : " + diskControllerType; + s_logger.debug(message); + throw new Exception(message); + } + + String newAdapterType = vmdkAdapterType.toString(); Pair vmdkInfo = getVmdkFileInfo(vmdkFileName); VmdkFileDescriptor vmdkFileDescriptor = vmdkInfo.first(); boolean isVmfsSparseFile = vmdkFileDescriptor.isVmfsSparseFile(); @@ -1240,6 +1252,10 @@ public class VirtualMachineMO extends BaseMO { } } + public void attachDisk(String[] vmdkDatastorePathChain, ManagedObjectReference morDs) throws Exception { + attachDisk(vmdkDatastorePathChain, morDs, null); + } + public void attachDisk(String[] vmdkDatastorePathChain, ManagedObjectReference morDs, String diskController) throws Exception { if(s_logger.isTraceEnabled()) @@ -1262,24 +1278,20 @@ public class VirtualMachineMO extends BaseMO { controllerKey = getIDEControllerKey(ideDeviceCount); unitNumber = getFreeUnitNumberOnIDEController(controllerKey); } else { - controllerKey = getScsiDiskControllerKey(diskController); + if (StringUtils.isNotBlank(diskController)) { + controllerKey = getScsiDiskControllerKey(diskController); + } else { + controllerKey = getScsiDeviceControllerKey(); + } unitNumber = -1; } + synchronized (_mor.getValue().intern()) { VirtualDevice newDisk = VmwareHelper.prepareDiskDevice(this, null, controllerKey, vmdkDatastorePathChain, morDs, unitNumber, 1); - controllerKey = newDisk.getControllerKey(); - unitNumber = newDisk.getUnitNumber(); - VirtualDiskFlatVer2BackingInfo backingInfo = (VirtualDiskFlatVer2BackingInfo)newDisk.getBacking(); - String vmdkFileName = backingInfo.getFileName(); - DiskControllerType diskControllerType = DiskControllerType.getType(diskController); - VmdkAdapterType vmdkAdapterType = VmdkAdapterType.getAdapterType(diskControllerType); - if (vmdkAdapterType == VmdkAdapterType.none) { - String message = "Failed to attach disk due to invalid vmdk adapter type for vmdk file [" + - vmdkFileName + "] with controller : " + diskControllerType; - s_logger.debug(message); - throw new Exception(message); + if (StringUtils.isNotBlank(diskController)) { + String vmdkFileName = vmdkDatastorePathChain[0]; + updateVmdkAdapter(vmdkFileName, diskController); } - updateVmdkAdapter(vmdkFileName, vmdkAdapterType.toString()); VirtualMachineConfigSpec reConfigSpec = new VirtualMachineConfigSpec(); VirtualDeviceConfigSpec deviceConfigSpec = new VirtualDeviceConfigSpec(); @@ -1319,69 +1331,6 @@ public class VirtualMachineMO extends BaseMO { } - public void attachDisk(String[] vmdkDatastorePathChain, ManagedObjectReference morDs) throws Exception { - - if (s_logger.isTraceEnabled()) - s_logger.trace("vCenter API trace - attachDisk(). target MOR: " + _mor.getValue() + ", vmdkDatastorePath: " + new Gson().toJson(vmdkDatastorePathChain) + - ", datastore: " + morDs.getValue()); - - synchronized (_mor.getValue().intern()) { - VirtualDevice newDisk = VmwareHelper.prepareDiskDevice(this, null, getScsiDeviceControllerKey(), vmdkDatastorePathChain, morDs, -1, 1); - VirtualMachineConfigSpec reConfigSpec = new VirtualMachineConfigSpec(); - VirtualDeviceConfigSpec deviceConfigSpec = new VirtualDeviceConfigSpec(); - - deviceConfigSpec.setDevice(newDisk); - deviceConfigSpec.setOperation(VirtualDeviceConfigSpecOperation.ADD); - - reConfigSpec.getDeviceChange().add(deviceConfigSpec); - - ManagedObjectReference morTask = _context.getService().reconfigVMTask(_mor, reConfigSpec); - boolean result = _context.getVimClient().waitForTask(morTask); - - if (!result) { - if (s_logger.isTraceEnabled()) - s_logger.trace("vCenter API trace - attachDisk() done(failed)"); - throw new Exception("Failed to attach disk due to " + TaskMO.getTaskFailureInfo(_context, morTask)); - } - - _context.waitForTaskProgressDone(morTask); - } - - if (s_logger.isTraceEnabled()) - s_logger.trace("vCenter API trace - attachDisk() done(successfully)"); - } - - public void attachDisk(Pair[] vmdkDatastorePathChain, int controllerKey) throws Exception { - - if (s_logger.isTraceEnabled()) - s_logger.trace("vCenter API trace - attachDisk(). target MOR: " + _mor.getValue() + ", vmdkDatastorePath: " + new Gson().toJson(vmdkDatastorePathChain)); - - synchronized (_mor.getValue().intern()) { - VirtualDevice newDisk = VmwareHelper.prepareDiskDevice(this, controllerKey, vmdkDatastorePathChain, -1, 1); - VirtualMachineConfigSpec reConfigSpec = new VirtualMachineConfigSpec(); - VirtualDeviceConfigSpec deviceConfigSpec = new VirtualDeviceConfigSpec(); - - deviceConfigSpec.setDevice(newDisk); - deviceConfigSpec.setOperation(VirtualDeviceConfigSpecOperation.ADD); - - reConfigSpec.getDeviceChange().add(deviceConfigSpec); - - ManagedObjectReference morTask = _context.getService().reconfigVMTask(_mor, reConfigSpec); - boolean result = _context.getVimClient().waitForTask(morTask); - - if (!result) { - if (s_logger.isTraceEnabled()) - s_logger.trace("vCenter API trace - attachDisk() done(failed)"); - throw new Exception("Failed to attach disk due to " + TaskMO.getTaskFailureInfo(_context, morTask)); - } - - _context.waitForTaskProgressDone(morTask); - } - - if (s_logger.isTraceEnabled()) - s_logger.trace("vCenter API trace - attachDisk() done(successfully)"); - } - // vmdkDatastorePath: [datastore name] vmdkFilePath public List> detachDisk(String vmdkDatastorePath, boolean deleteBackingFile) throws Exception { @@ -2489,7 +2438,7 @@ public class VirtualMachineMO extends BaseMO { s_logger.info("Disk backing : " + diskBackingInfo.getFileName() + " matches ==> " + deviceNumbering); if (((VirtualDisk) device).getVDiskId() == null) { s_logger.debug("vDiskid does not exist for volume " + vmdkDatastorePath + " registering the disk now"); - VirtualStorageObjectManager vStorageObjectManagerMO = new VirtualStorageObjectManager(getOwnerDatacenter().first().getContext()); + VirtualStorageObjectManagerMO vStorageObjectManagerMO = new VirtualStorageObjectManagerMO(getOwnerDatacenter().first().getContext()); VStorageObject vStorageObject = vStorageObjectManagerMO.registerVirtualDisk(dsBackingFile, null, getOwnerDatacenter().first().getName()); VStorageObjectConfigInfo diskConfigInfo = vStorageObject.getConfig(); ((VirtualDisk) device).setVDiskId(diskConfigInfo.getId()); diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualStorageObjectManager.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualStorageObjectManager.java deleted file mode 100644 index 0b6f44ae0b0..00000000000 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualStorageObjectManager.java +++ /dev/null @@ -1,58 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -package com.cloud.hypervisor.vmware.mo; - -import com.vmware.vim25.ID; -import com.vmware.vim25.VStorageObject; -import org.apache.log4j.Logger; - -import com.vmware.vim25.ManagedObjectReference; - -import com.cloud.hypervisor.vmware.util.VmwareContext; - -public class VirtualStorageObjectManager extends BaseMO { - @SuppressWarnings("unused") - private static final Logger s_logger = Logger.getLogger(VirtualStorageObjectManager.class); - - public VirtualStorageObjectManager(VmwareContext context) { - super(context, context.getServiceContent().getVStorageObjectManager()); - } - - public VirtualStorageObjectManager(VmwareContext context, ManagedObjectReference morDiskMgr) { - super(context, morDiskMgr); - } - - public VirtualStorageObjectManager(VmwareContext context, String morType, String morValue) { - super(context, morType, morValue); - } - - public VStorageObject registerVirtualDisk(DatastoreFile datastoreFile, String name, String dcName) throws Exception { - StringBuilder sb = new StringBuilder(); - //https://10.2.2.254/folder/i-2-4-VM/89e3756d9b7444dc92388eb36ddd026b.vmdk?dcPath=datacenter-21&dsName=c84e4af9b6ac33e887a25d9242650091 - sb.append("https://").append(_context.getServerAddress()).append("/folder/"); - sb.append(datastoreFile.getRelativePath()); - sb.append("?dcPath="); - sb.append(dcName); - sb.append("&dsName="); - sb.append(datastoreFile.getDatastoreName()); - return _context.getService().registerDisk(_mor, sb.toString(), name); - } - - public VStorageObject retrieveVirtualDisk (ID id, ManagedObjectReference morDS) throws Exception { - return _context.getService().retrieveVStorageObject(_mor, id, morDS); - } -} diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualStorageObjectManagerMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualStorageObjectManagerMO.java new file mode 100644 index 00000000000..7924dfda646 --- /dev/null +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualStorageObjectManagerMO.java @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.hypervisor.vmware.mo; + +import com.vmware.vim25.ID; +import com.vmware.vim25.TaskInfo; +import com.vmware.vim25.VStorageObject; +import com.vmware.vim25.VirtualDiskType; +import com.vmware.vim25.VslmCreateSpec; +import com.vmware.vim25.VslmCreateSpecDiskFileBackingSpec; +import org.apache.log4j.Logger; + +import com.vmware.vim25.ManagedObjectReference; + +import com.cloud.hypervisor.vmware.util.VmwareContext; + +public class VirtualStorageObjectManagerMO extends BaseMO { + @SuppressWarnings("unused") + private static final Logger s_logger = Logger.getLogger(VirtualStorageObjectManagerMO.class); + + public VirtualStorageObjectManagerMO(VmwareContext context) { + super(context, context.getServiceContent().getVStorageObjectManager()); + } + + public VirtualStorageObjectManagerMO(VmwareContext context, ManagedObjectReference morDiskMgr) { + super(context, morDiskMgr); + } + + public VirtualStorageObjectManagerMO(VmwareContext context, String morType, String morValue) { + super(context, morType, morValue); + } + + public VStorageObject registerVirtualDisk(DatastoreFile datastoreFile, String name, String dcName) throws Exception { + StringBuilder sb = new StringBuilder(); + //https://10.2.2.254/folder/i-2-4-VM/89e3756d9b7444dc92388eb36ddd026b.vmdk?dcPath=datacenter-21&dsName=c84e4af9b6ac33e887a25d9242650091 + sb.append("https://").append(_context.getServerAddress()).append("/folder/"); + sb.append(datastoreFile.getRelativePath()); + sb.append("?dcPath="); + sb.append(dcName); + sb.append("&dsName="); + sb.append(datastoreFile.getDatastoreName()); + return _context.getService().registerDisk(_mor, sb.toString(), name); + } + + public VStorageObject retrieveVirtualDisk (ID id, ManagedObjectReference morDS) throws Exception { + return _context.getService().retrieveVStorageObject(_mor, id, morDS); + } + + public VStorageObject createDisk(ManagedObjectReference morDS, VirtualDiskType diskType, long currentSizeInBytes, String datastoreFilepath, String filename) throws Exception { + long currentSizeInMB = currentSizeInBytes/(1024*1024); + + VslmCreateSpecDiskFileBackingSpec diskFileBackingSpec = new VslmCreateSpecDiskFileBackingSpec(); + diskFileBackingSpec.setDatastore(morDS); + diskFileBackingSpec.setProvisioningType(diskType.value()); + // path should be just the folder name. For example, instead of '[datastore1] folder1/filename.vmdk' you would just do 'folder1'. + // path is introduced from 6.7. In 6.5 disk will be created in the default folder "fcd" + diskFileBackingSpec.setPath(null); + + VslmCreateSpec vslmCreateSpec = new VslmCreateSpec(); + vslmCreateSpec.setBackingSpec(diskFileBackingSpec); + vslmCreateSpec.setCapacityInMB(currentSizeInMB); + vslmCreateSpec.setName(filename); + + ManagedObjectReference morTask = _context.getService().createDiskTask(_mor, vslmCreateSpec); + boolean result = _context.getVimClient().waitForTask(morTask); + + VStorageObject vStorageObject = null; + if (result) { + _context.waitForTaskProgressDone(morTask); + //_context.getService().reconcileDatastoreInventoryTask(_mor, morDS); + TaskInfo taskInfo = TaskMO.getTaskInfo(_context, morTask); + vStorageObject = (VStorageObject)taskInfo.getResult(); + + } else { + s_logger.error("VMware CreateDisk_Task failed due to " + TaskMO.getTaskFailureInfo(_context, morTask)); + } + + return vStorageObject; + } +} diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java index 181b2ef183f..08b5672760a 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java @@ -212,35 +212,49 @@ public class VmwareHelper { } // vmdkDatastorePath: [datastore name] vmdkFilePath - public static VirtualDevice prepareDiskDevice(VirtualMachineMO vmMo, int controllerKey, String vmdkDatastorePath, int sizeInMb, ManagedObjectReference morDs, - int deviceNumber, int contextNumber) throws Exception { + public static VirtualDevice prepareDiskDevice(VirtualMachineMO vmMo, VirtualDisk device, int controllerKey, String vmdkDatastorePathChain[], + ManagedObjectReference morDs, int deviceNumber, int contextNumber) throws Exception { - VirtualDisk disk = new VirtualDisk(); + assert (vmdkDatastorePathChain != null); + assert (vmdkDatastorePathChain.length >= 1); - VirtualDiskFlatVer2BackingInfo backingInfo = new VirtualDiskFlatVer2BackingInfo(); - backingInfo.setDiskMode(VirtualDiskMode.PERSISTENT.value()); - backingInfo.setThinProvisioned(true); - backingInfo.setEagerlyScrub(false); - backingInfo.setDatastore(morDs); - backingInfo.setFileName(vmdkDatastorePath); - disk.setBacking(backingInfo); + VirtualDisk disk; + VirtualDiskFlatVer2BackingInfo backingInfo; + if (device != null) { + disk = device; + backingInfo = (VirtualDiskFlatVer2BackingInfo)disk.getBacking(); + } else { + disk = new VirtualDisk(); + backingInfo = new VirtualDiskFlatVer2BackingInfo(); + backingInfo.setDatastore(morDs); + backingInfo.setDiskMode(VirtualDiskMode.PERSISTENT.value()); + disk.setBacking(backingInfo); - int ideControllerKey = vmMo.getIDEDeviceControllerKey(); - if (controllerKey < 0) - controllerKey = ideControllerKey; - if (deviceNumber < 0) { - deviceNumber = vmMo.getNextDeviceNumber(controllerKey); + int ideControllerKey = vmMo.getIDEDeviceControllerKey(); + if (controllerKey < 0) + controllerKey = ideControllerKey; + if (deviceNumber < 0) { + deviceNumber = vmMo.getNextDeviceNumber(controllerKey); + } + + disk.setControllerKey(controllerKey); + disk.setKey(-contextNumber); + disk.setUnitNumber(deviceNumber); + + VirtualDeviceConnectInfo connectInfo = new VirtualDeviceConnectInfo(); + connectInfo.setConnected(true); + connectInfo.setStartConnected(true); + disk.setConnectable(connectInfo); } - disk.setControllerKey(controllerKey); - disk.setKey(-contextNumber); - disk.setUnitNumber(deviceNumber); - disk.setCapacityInKB(sizeInMb * 1024); + backingInfo.setFileName(vmdkDatastorePathChain[0]); + if (vmdkDatastorePathChain.length > 1) { + String[] parentDisks = new String[vmdkDatastorePathChain.length - 1]; + for (int i = 0; i < vmdkDatastorePathChain.length - 1; i++) + parentDisks[i] = vmdkDatastorePathChain[i + 1]; - VirtualDeviceConnectInfo connectInfo = new VirtualDeviceConnectInfo(); - connectInfo.setConnected(true); - connectInfo.setStartConnected(true); - disk.setConnectable(connectInfo); + setParentBackingInfo(backingInfo, morDs, parentDisks); + } return disk; } @@ -314,96 +328,6 @@ public class VmwareHelper { return disk; } - // vmdkDatastorePath: [datastore name] vmdkFilePath - public static VirtualDevice prepareDiskDevice(VirtualMachineMO vmMo, VirtualDisk device, int controllerKey, String vmdkDatastorePathChain[], - ManagedObjectReference morDs, int deviceNumber, int contextNumber) throws Exception { - - assert (vmdkDatastorePathChain != null); - assert (vmdkDatastorePathChain.length >= 1); - - VirtualDisk disk; - VirtualDiskFlatVer2BackingInfo backingInfo; - if (device != null) { - disk = device; - backingInfo = (VirtualDiskFlatVer2BackingInfo)disk.getBacking(); - } else { - disk = new VirtualDisk(); - backingInfo = new VirtualDiskFlatVer2BackingInfo(); - backingInfo.setDatastore(morDs); - backingInfo.setDiskMode(VirtualDiskMode.PERSISTENT.value()); - disk.setBacking(backingInfo); - - int ideControllerKey = vmMo.getIDEDeviceControllerKey(); - if (controllerKey < 0) - controllerKey = ideControllerKey; - if (deviceNumber < 0) { - deviceNumber = vmMo.getNextDeviceNumber(controllerKey); - } - - disk.setControllerKey(controllerKey); - disk.setKey(-contextNumber); - disk.setUnitNumber(deviceNumber); - - VirtualDeviceConnectInfo connectInfo = new VirtualDeviceConnectInfo(); - connectInfo.setConnected(true); - connectInfo.setStartConnected(true); - disk.setConnectable(connectInfo); - } - - backingInfo.setFileName(vmdkDatastorePathChain[0]); - if (vmdkDatastorePathChain.length > 1) { - String[] parentDisks = new String[vmdkDatastorePathChain.length - 1]; - for (int i = 0; i < vmdkDatastorePathChain.length - 1; i++) - parentDisks[i] = vmdkDatastorePathChain[i + 1]; - - setParentBackingInfo(backingInfo, morDs, parentDisks); - } - - return disk; - } - - @SuppressWarnings("unchecked") - public static VirtualDevice prepareDiskDevice(VirtualMachineMO vmMo, int controllerKey, Pair[] vmdkDatastorePathChain, - int deviceNumber, int contextNumber) throws Exception { - - assert (vmdkDatastorePathChain != null); - assert (vmdkDatastorePathChain.length >= 1); - - VirtualDisk disk = new VirtualDisk(); - - VirtualDiskFlatVer2BackingInfo backingInfo = new VirtualDiskFlatVer2BackingInfo(); - backingInfo.setDatastore(vmdkDatastorePathChain[0].second()); - backingInfo.setFileName(vmdkDatastorePathChain[0].first()); - backingInfo.setDiskMode(VirtualDiskMode.PERSISTENT.value()); - if (vmdkDatastorePathChain.length > 1) { - Pair[] parentDisks = new Pair[vmdkDatastorePathChain.length - 1]; - for (int i = 0; i < vmdkDatastorePathChain.length - 1; i++) - parentDisks[i] = vmdkDatastorePathChain[i + 1]; - - setParentBackingInfo(backingInfo, parentDisks); - } - - disk.setBacking(backingInfo); - - int ideControllerKey = vmMo.getIDEDeviceControllerKey(); - if (controllerKey < 0) - controllerKey = ideControllerKey; - if (deviceNumber < 0) { - deviceNumber = vmMo.getNextDeviceNumber(controllerKey); - } - - disk.setControllerKey(controllerKey); - disk.setKey(-contextNumber); - disk.setUnitNumber(deviceNumber); - - VirtualDeviceConnectInfo connectInfo = new VirtualDeviceConnectInfo(); - connectInfo.setConnected(true); - connectInfo.setStartConnected(true); - disk.setConnectable(connectInfo); - - return disk; - } - private static void setParentBackingInfo(VirtualDiskFlatVer2BackingInfo backingInfo, ManagedObjectReference morDs, String[] parentDatastorePathList) { VirtualDiskFlatVer2BackingInfo parentBacking = new VirtualDiskFlatVer2BackingInfo(); From bef00b1cfb4df88c0778bbc3841650613e3ee025 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Fri, 26 Jun 2020 13:33:45 +0530 Subject: [PATCH 016/164] Added PBM jar dependency for vsphere 6.7 version --- deps/install-non-oss.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/deps/install-non-oss.sh b/deps/install-non-oss.sh index f1c63c4a95f..01da6dab93f 100755 --- a/deps/install-non-oss.sh +++ b/deps/install-non-oss.sh @@ -41,4 +41,7 @@ mvn install:install-file -Dfile=vim25_65.jar -DgroupId=com.cloud.com.vmwa mvn install:install-file -Dfile=vim25_67.jar -DgroupId=com.cloud.com.vmware -DartifactId=vmware-vim25 -Dversion=6.7 -Dpackaging=jar # From https://my.vmware.com/group/vmware/get-download?downloadGroup=VS-MGMT-SDK65 -mvn install:install-file -Dfile=pbm_65.jar -DgroupId=com.cloud.com.vmware -DartifactId=pbm -Dversion=6.5 -Dpackaging=jar +mvn install:install-file -Dfile=pbm_65.jar -DgroupId=com.cloud.com.vmware -DartifactId=vmware-pbm -Dversion=6.5 -Dpackaging=jar + +# From https://my.vmware.com/group/vmware/details?downloadGroup=WEBCLIENTSDK67U2&productId=742 +mvn install:install-file -Dfile=pbm_67.jar -DgroupId=com.cloud.com.vmware -DartifactId=vmware-pbm -Dversion=6.7 -Dpackaging=jar From 3d9edeeb9d2ecaba06573980dc48225cc126d628 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Fri, 3 Jul 2020 10:39:32 +0530 Subject: [PATCH 017/164] Bug fixing on default base folder --- .../resource/VmwareStorageLayoutHelper.java | 54 +++++++++++++++---- .../resource/VmwareStorageProcessor.java | 4 +- .../hypervisor/vmware/mo/DatastoreMO.java | 41 -------------- .../vmware/mo/HypervisorHostHelper.java | 3 +- .../vmware/mo/VirtualMachineMO.java | 1 + .../hypervisor/vmware/util/VmwareHelper.java | 1 + 6 files changed, 50 insertions(+), 54 deletions(-) diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageLayoutHelper.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageLayoutHelper.java index 91970e9073b..d10c63bb3dd 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageLayoutHelper.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageLayoutHelper.java @@ -51,12 +51,45 @@ public class VmwareStorageLayoutHelper { return filePair; case CLOUDSTACK_LEGACY: - filePair[0] = getLegacyDatastorePathFromVmdkFileName(dsMo, vmdkName + ".vmdk"); + filePair[0] = getDatastorePathBaseFolderFromVmdkFileName(dsMo, vmdkName + ".vmdk"); if (linkedVmdk) - filePair[1] = getLegacyDatastorePathFromVmdkFileName(dsMo, vmdkName + "-delta.vmdk"); + filePair[1] = getDatastorePathBaseFolderFromVmdkFileName(dsMo, vmdkName + "-delta.vmdk"); else - filePair[1] = getLegacyDatastorePathFromVmdkFileName(dsMo, vmdkName + "-flat.vmdk"); + filePair[1] = getDatastorePathBaseFolderFromVmdkFileName(dsMo, vmdkName + "-flat.vmdk"); + return filePair; + + default: + assert (false); + break; + } + + assert (false); + return null; + } + + public static String[] getVmdkFilePairManagedDatastorePath(DatastoreMO dsMo, String vmName, String vmdkName, VmwareStorageLayoutType layoutType, boolean linkedVmdk) + throws Exception { + + String[] filePair = new String[2]; + switch (layoutType) { + case VMWARE: + assert (vmName != null && !vmName.isEmpty()); + filePair[0] = getVmwareDatastorePathFromVmdkFileName(dsMo, vmName, vmdkName + ".vmdk"); + + if (linkedVmdk) + filePair[1] = getVmwareDatastorePathFromVmdkFileName(dsMo, vmName, vmdkName + "-delta.vmdk"); + else + filePair[1] = getVmwareDatastorePathFromVmdkFileName(dsMo, vmName, vmdkName + "-flat.vmdk"); + return filePair; + + case CLOUDSTACK_LEGACY: + filePair[0] = getDeprecatedLegacyDatastorePathFromVmdkFileName(dsMo, vmdkName + ".vmdk"); + + if (linkedVmdk) + filePair[1] = getDeprecatedLegacyDatastorePathFromVmdkFileName(dsMo, vmdkName + "-delta.vmdk"); + else + filePair[1] = getDeprecatedLegacyDatastorePathFromVmdkFileName(dsMo, vmdkName + "-flat.vmdk"); return filePair; default: @@ -161,7 +194,7 @@ public class VmwareStorageLayoutHelper { DatastoreFile srcDsFile = new DatastoreFile(fileDsFullPath); String companionFilePath = srcDsFile.getCompanionPath(vmdkName + "-flat.vmdk"); if (ds.fileExists(companionFilePath)) { - String targetPath = getLegacyDatastorePathFromVmdkFileName(ds, vmdkName + "-flat.vmdk"); + String targetPath = getDatastorePathBaseFolderFromVmdkFileName(ds, vmdkName + "-flat.vmdk"); s_logger.info("Fixup folder-synchronization. move " + companionFilePath + " -> " + targetPath); ds.moveDatastoreFile(companionFilePath, dcMo.getMor(), ds.getMor(), targetPath, dcMo.getMor(), true); @@ -169,14 +202,14 @@ public class VmwareStorageLayoutHelper { companionFilePath = srcDsFile.getCompanionPath(vmdkName + "-delta.vmdk"); if (ds.fileExists(companionFilePath)) { - String targetPath = getLegacyDatastorePathFromVmdkFileName(ds, vmdkName + "-delta.vmdk"); + String targetPath = getDatastorePathBaseFolderFromVmdkFileName(ds, vmdkName + "-delta.vmdk"); s_logger.info("Fixup folder-synchronization. move " + companionFilePath + " -> " + targetPath); ds.moveDatastoreFile(companionFilePath, dcMo.getMor(), ds.getMor(), targetPath, dcMo.getMor(), true); } // move the identity VMDK file the last - String targetPath = getLegacyDatastorePathFromVmdkFileName(ds, vmdkName + ".vmdk"); + String targetPath = getDatastorePathBaseFolderFromVmdkFileName(ds, vmdkName + ".vmdk"); s_logger.info("Fixup folder-synchronization. move " + fileDsFullPath + " -> " + targetPath); ds.moveDatastoreFile(fileDsFullPath, dcMo.getMor(), ds.getMor(), targetPath, dcMo.getMor(), true); @@ -310,17 +343,20 @@ public class VmwareStorageLayoutHelper { } } + //This method call is for the volumes which actually exists public static String getLegacyDatastorePathFromVmdkFileName(DatastoreMO dsMo, String vmdkFileName) throws Exception { String vmdkDatastorePath = String.format("[%s] %s/%s", dsMo.getName(), HypervisorHostHelper.VSPHERE_DATASTORE_BASE_FOLDER, vmdkFileName); - if (!dsMo.fileExists(vmdkDatastorePath)) { - vmdkDatastorePath = String.format("[%s] %s/%s", dsMo.getName(), HypervisorHostHelper.VSPHERE_FCD_DEFAULT_FOLDER, vmdkFileName); - } if (!dsMo.fileExists(vmdkDatastorePath)) { vmdkDatastorePath = getDeprecatedLegacyDatastorePathFromVmdkFileName(dsMo, vmdkFileName); } return vmdkDatastorePath; } + //This method call is for the volumes to be created or can also be for volumes already exists + public static String getDatastorePathBaseFolderFromVmdkFileName(DatastoreMO dsMo, String vmdkFileName) throws Exception { + return String.format("[%s] %s/%s", dsMo.getName(), HypervisorHostHelper.VSPHERE_DATASTORE_BASE_FOLDER, vmdkFileName); + } + public static String getDeprecatedLegacyDatastorePathFromVmdkFileName(DatastoreMO dsMo, String vmdkFileName) throws Exception { return String.format("[%s] %s", dsMo.getName(), vmdkFileName); } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java index 0e12d6e1d75..9cd9b6e9ea3 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -665,9 +665,9 @@ public class VmwareStorageProcessor implements StorageProcessor { VirtualMachineMO vmMo = vmInfo.first(); vmMo.unregisterVm(); - String[] vmwareLayoutFilePair = VmwareStorageLayoutHelper.getVmdkFilePairDatastorePath(dsMo, managedStoragePoolRootVolumeName, + String[] vmwareLayoutFilePair = VmwareStorageLayoutHelper.getVmdkFilePairManagedDatastorePath(dsMo, managedStoragePoolRootVolumeName, managedStoragePoolRootVolumeName, VmwareStorageLayoutType.VMWARE, false); - String[] legacyCloudStackLayoutFilePair = VmwareStorageLayoutHelper.getVmdkFilePairDatastorePath(dsMo, null, + String[] legacyCloudStackLayoutFilePair = VmwareStorageLayoutHelper.getVmdkFilePairManagedDatastorePath(dsMo, null, managedStoragePoolRootVolumeName, VmwareStorageLayoutType.CLOUDSTACK_LEGACY, false); dsMo.moveDatastoreFile(vmwareLayoutFilePair[0], dcMo.getMor(), dsMo.getMor(), legacyCloudStackLayoutFilePair[0], dcMo.getMor(), true); diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatastoreMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatastoreMO.java index 3e914a4c277..b9eeaeda309 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatastoreMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatastoreMO.java @@ -279,47 +279,6 @@ public class DatastoreMO extends BaseMO { return false; } - public String[] getVmdkFileChain(String rootVmdkDatastoreFullPath) throws Exception { - Pair dcPair = getOwnerDatacenter(); - - List files = new ArrayList<>(); - files.add(rootVmdkDatastoreFullPath); - - String currentVmdkFullPath = rootVmdkDatastoreFullPath; - while (true) { - String url = getContext().composeDatastoreBrowseUrl(dcPair.second(), currentVmdkFullPath); - byte[] content = getContext().getResourceContent(url); - if (content == null || content.length == 0) - break; - - VmdkFileDescriptor descriptor = new VmdkFileDescriptor(); - descriptor.parse(content); - - String parentFileName = descriptor.getParentFileName(); - if (parentFileName == null) - break; - - if (parentFileName.startsWith("/")) { - // when parent file is not at the same directory as it is, assume it is at parent directory - // this is only valid in Apache CloudStack primary storage deployment - DatastoreFile dsFile = new DatastoreFile(currentVmdkFullPath); - String dir = dsFile.getDir(); - if (dir != null && dir.lastIndexOf('/') > 0) - dir = dir.substring(0, dir.lastIndexOf('/')); - else - dir = ""; - - currentVmdkFullPath = new DatastoreFile(dsFile.getDatastoreName(), dir, parentFileName.substring(parentFileName.lastIndexOf('/') + 1)).getPath(); - files.add(currentVmdkFullPath); - } else { - currentVmdkFullPath = DatastoreFile.getCompanionDatastorePath(currentVmdkFullPath, parentFileName); - files.add(currentVmdkFullPath); - } - } - - return files.toArray(new String[0]); - } - @Deprecated public String[] listDirContent(String path) throws Exception { String fullPath = path; diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java index 6fd1a31c865..f25279bdee3 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java @@ -138,8 +138,7 @@ public class HypervisorHostHelper { private static final String UNTAGGED_VLAN_NAME = "untagged"; private static final String VMDK_PACK_DIR = "ova"; private static final String OVA_OPTION_KEY_BOOTDISK = "cloud.ova.bootdisk"; - public static final String VSPHERE_DATASTORE_BASE_FOLDER = ".cloudstack.base.folder"; - public static final String VSPHERE_FCD_DEFAULT_FOLDER = "fcd"; + public static final String VSPHERE_DATASTORE_BASE_FOLDER = "fcd"; public static VirtualMachineMO findVmFromObjectContent(VmwareContext context, ObjectContent[] ocs, String name, String instanceNameCustomField) { diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java index 3100b7f26ff..b2af66116ee 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java @@ -36,6 +36,7 @@ import java.util.concurrent.Future; import com.vmware.vim25.VStorageObject; import com.vmware.vim25.VStorageObjectConfigInfo; +import com.vmware.vim25.VirtualMachineRelocateTransformation; import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; import org.apache.commons.lang.StringUtils; diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java index 08b5672760a..e2513ad6fb5 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java @@ -36,6 +36,7 @@ import javax.xml.datatype.DatatypeConfigurationException; import javax.xml.datatype.DatatypeFactory; import javax.xml.datatype.XMLGregorianCalendar; +import com.vmware.vim25.VirtualDiskType; import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; From 4bbb747b2300d8fa60e04ba3ab210020637be80f Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Wed, 1 Jul 2020 09:48:42 +0530 Subject: [PATCH 018/164] VMFS to presetup change --- .../cloud/hypervisor/vmware/resource/VmwareResource.java | 6 +++--- .../com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 0e13c309081..50b95c14a72 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -4414,7 +4414,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } // If datastore is VMFS and target datastore is not mounted or accessible to source host then fail migration. - if (filerTo.getType().equals(StoragePoolType.VMFS)) { + if (filerTo.getType().equals(StoragePoolType.VMFS) || filerTo.getType().equals(StoragePoolType.PreSetup)) { if (morDsAtSource == null) { s_logger.warn( "If host version is below 5.1, then target VMFS datastore(s) need to manually mounted on source host for a successful live storage migration."); @@ -4901,7 +4901,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa VmwareHypervisorHost hyperHost = getHyperHost(getServiceContext()); StorageFilerTO pool = cmd.getPool(); - if (pool.getType() != StoragePoolType.NetworkFilesystem && pool.getType() != StoragePoolType.VMFS) { + if (pool.getType() != StoragePoolType.NetworkFilesystem && pool.getType() != StoragePoolType.VMFS && pool.getType() != StoragePoolType.PreSetup) { throw new Exception("Unsupported storage pool type " + pool.getType()); } @@ -4923,7 +4923,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa Map tInfo = new HashMap<>(); ModifyStoragePoolAnswer answer = new ModifyStoragePoolAnswer(cmd, capacity, available, tInfo); - if (cmd.getAdd() && pool.getType() == StoragePoolType.VMFS) { + if (cmd.getAdd() && (pool.getType() == StoragePoolType.VMFS || pool.getType() == StoragePoolType.PreSetup)) { answer.setLocalDatastoreName(morDatastore.getValue()); } diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java index b2af66116ee..3100b7f26ff 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java @@ -36,7 +36,6 @@ import java.util.concurrent.Future; import com.vmware.vim25.VStorageObject; import com.vmware.vim25.VStorageObjectConfigInfo; -import com.vmware.vim25.VirtualMachineRelocateTransformation; import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; import org.apache.commons.lang.StringUtils; From 6df819028e472dd9801873857fe20133381ca0fe Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Mon, 6 Jul 2020 09:56:29 +0530 Subject: [PATCH 019/164] UI changes and accept any type of datastore as presetup in vmware --- .../cloud/agent/api/ModifyStoragePoolAnswer.java | 10 ++++++++++ .../datastore/provider/DefaultHostListener.java | 9 +++++++++ .../hypervisor/vmware/resource/VmwareResource.java | 3 ++- .../CloudStackPrimaryDataStoreLifeCycleImpl.java | 4 +++- .../api/query/dao/StoragePoolJoinDaoImpl.java | 9 +++++++++ .../java/com/cloud/storage/StorageManagerImpl.java | 5 +++++ .../cloudstack/vm/UnmanagedVMsManagerImpl.java | 2 +- ui/scripts/docs.js | 2 +- ui/scripts/system.js | 14 +++++++------- ui/scripts/zoneWizard.js | 14 +++++++------- .../cloud/hypervisor/vmware/mo/DatastoreMO.java | 5 +++++ .../cloud/hypervisor/vmware/util/VmwareHelper.java | 1 - 12 files changed, 59 insertions(+), 19 deletions(-) diff --git a/core/src/main/java/com/cloud/agent/api/ModifyStoragePoolAnswer.java b/core/src/main/java/com/cloud/agent/api/ModifyStoragePoolAnswer.java index 6e6dadc67f7..a5f7c53214b 100644 --- a/core/src/main/java/com/cloud/agent/api/ModifyStoragePoolAnswer.java +++ b/core/src/main/java/com/cloud/agent/api/ModifyStoragePoolAnswer.java @@ -27,6 +27,7 @@ public class ModifyStoragePoolAnswer extends Answer { private StoragePoolInfo poolInfo; private Map templateInfo; private String localDatastoreName; + private String poolType; public ModifyStoragePoolAnswer(ModifyStoragePoolCommand cmd, long capacityBytes, long availableBytes, Map tInfo) { super(cmd); @@ -61,4 +62,13 @@ public class ModifyStoragePoolAnswer extends Answer { public String getLocalDatastoreName() { return localDatastoreName; } + + + public String getPoolType() { + return poolType; + } + + public void setPoolType(String poolType) { + this.poolType = poolType; + } } diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java index 64533d54d2f..1b9682a2fa3 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java @@ -22,6 +22,9 @@ import java.util.List; import javax.inject.Inject; +import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailVO; +import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; +import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; @@ -53,6 +56,8 @@ public class DefaultHostListener implements HypervisorHostListener { StoragePoolHostDao storagePoolHostDao; @Inject PrimaryDataStoreDao primaryStoreDao; + @Inject + StoragePoolDetailsDao storagePoolDetailsDao; @Override public boolean hostAdded(long hostId) { @@ -102,6 +107,10 @@ public class DefaultHostListener implements HypervisorHostListener { StoragePoolVO poolVO = this.primaryStoreDao.findById(poolId); poolVO.setUsedBytes(mspAnswer.getPoolInfo().getCapacityBytes() - mspAnswer.getPoolInfo().getAvailableBytes()); poolVO.setCapacityBytes(mspAnswer.getPoolInfo().getCapacityBytes()); + if(StringUtils.isNotEmpty(mspAnswer.getPoolType())) { + StoragePoolDetailVO storagePoolDetailVO = new StoragePoolDetailVO(poolId, "pool_type", mspAnswer.getPoolType(), false); + storagePoolDetailsDao.persist(storagePoolDetailVO); + } primaryStoreDao.update(pool.getId(), poolVO); s_logger.info("Connection established between storage pool " + pool + " and host " + hostId); diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 50b95c14a72..1230fa8fe1a 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -4908,7 +4908,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa ManagedObjectReference morDatastore = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, pool.getUuid()); if (morDatastore == null) { - morDatastore = hyperHost.mountDatastore(pool.getType() == StoragePoolType.VMFS, pool.getHost(), pool.getPort(), pool.getPath(), pool.getUuid().replace("-", "")); + morDatastore = hyperHost.mountDatastore((pool.getType() == StoragePoolType.VMFS || pool.getType() == StoragePoolType.PreSetup), pool.getHost(), pool.getPort(), pool.getPath(), pool.getUuid().replace("-", "")); } assert (morDatastore != null); @@ -4924,6 +4924,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa ModifyStoragePoolAnswer answer = new ModifyStoragePoolAnswer(cmd, capacity, available, tInfo); if (cmd.getAdd() && (pool.getType() == StoragePoolType.VMFS || pool.getType() == StoragePoolType.PreSetup)) { + answer.setPoolType(dsMo.getDatastoreType()); answer.setLocalDatastoreName(morDatastore.getValue()); } diff --git a/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java b/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java index bdaeddca4c0..84b7bfdf9af 100644 --- a/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java +++ b/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java @@ -29,6 +29,7 @@ import java.util.UUID; import javax.inject.Inject; +import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope; @@ -133,6 +134,7 @@ public class CloudStackPrimaryDataStoreLifeCycleImpl implements PrimaryDataStore Long zoneId = (Long)dsInfos.get("zoneId"); String url = (String)dsInfos.get("url"); String providerName = (String)dsInfos.get("providerName"); + String hypervisorType = (String)dsInfos.get("hypervisorType"); if (clusterId != null && podId == null) { throw new InvalidParameterValueException("Cluster id requires pod id"); } @@ -327,7 +329,7 @@ public class CloudStackPrimaryDataStoreLifeCycleImpl implements PrimaryDataStore uuid = (String)existingUuid; } else if (scheme.equalsIgnoreCase("sharedmountpoint") || scheme.equalsIgnoreCase("clvm")) { uuid = UUID.randomUUID().toString(); - } else if (scheme.equalsIgnoreCase("PreSetup")) { + } else if (scheme.equalsIgnoreCase("PreSetup") && !(StringUtils.isNotBlank(hypervisorType) && HypervisorType.getType(hypervisorType).equals(HypervisorType.VMware))) { uuid = hostPath.replace("/", ""); } else { uuid = UUID.nameUUIDFromBytes((storageHost + hostPath).getBytes()).toString(); diff --git a/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java index b8b312bd267..a248d71e72c 100644 --- a/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java @@ -32,6 +32,8 @@ import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailVO; +import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -52,6 +54,9 @@ public class StoragePoolJoinDaoImpl extends GenericDaoBase spSearch; private final SearchBuilder spIdSearch; @@ -94,6 +99,10 @@ public class StoragePoolJoinDaoImpl extends GenericDaoBase details = extractApiParamAsMap(cmd.getDetails()); @@ -716,6 +720,7 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C params.put("zoneId", zone.getId()); params.put("clusterId", clusterId); params.put("podId", podId); + params.put("hypervisorType", hypervisorType.toString()); params.put("url", cmd.getUrl()); params.put("tags", cmd.getTags()); params.put("name", cmd.getStoragePoolName()); diff --git a/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java b/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java index 26f8675f226..0d614349235 100644 --- a/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java @@ -491,7 +491,7 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager { final String dsPath = disk.getDatastorePath(); final String dsType = disk.getDatastoreType(); final String dsName = disk.getDatastoreName(); - if (dsType.equals("VMFS")) { + if (dsType.equals(Storage.StoragePoolType.VMFS.toString())) { List pools = primaryDataStoreDao.listPoolsByCluster(cluster.getId()); pools.addAll(primaryDataStoreDao.listByDataCenterId(zone.getId())); for (StoragePool pool : pools) { diff --git a/ui/scripts/docs.js b/ui/scripts/docs.js index 7f29f2b3ac2..a801a6b52e0 100755 --- a/ui/scripts/docs.js +++ b/ui/scripts/docs.js @@ -711,7 +711,7 @@ cloudStack.docs = { externalLink: '' }, helpPrimaryStorageProtocol: { - desc: 'For XenServer, choose NFS, iSCSI, or PreSetup. For KVM, choose NFS, SharedMountPoint, RDB, CLVM or Gluster. For vSphere, choose VMFS (iSCSI or FiberChannel) or NFS. For Hyper-V, choose SMB/CIFS. For LXC, choose NFS or SharedMountPoint. For OVM, choose NFS or ocfs2.', + desc: 'For XenServer, choose NFS, iSCSI, or PreSetup. For KVM, choose NFS, SharedMountPoint, RDB, CLVM or Gluster. For vSphere, choose PreSetup (VMFS or iSCSI or FiberChannel or vSAN or vVols) or NFS. For Hyper-V, choose SMB/CIFS. For LXC, choose NFS or SharedMountPoint. For OVM, choose NFS or ocfs2.', externalLink: '' }, helpPrimaryStorageServer: { diff --git a/ui/scripts/system.js b/ui/scripts/system.js index 418ce53bc4c..ca0eb7e3aeb 100755 --- a/ui/scripts/system.js +++ b/ui/scripts/system.js @@ -18735,8 +18735,8 @@ description: "nfs" }); items.push({ - id: "vmfs", - description: "vmfs" + id: "presetup", + description: "presetup" }); items.push({ id: "custom", @@ -18885,7 +18885,7 @@ $form.find('.form-item[rel=rbdsecret]').hide(); $form.find('.form-item[rel=glustervolume]').hide(); - } else if (protocol == "PreSetup") { + } else if (protocol == "PreSetup" && selectedClusterObj.hypervisortype != "VMware") { $form.find('.form-item[rel=server]').hide(); $form.find('.form-item[rel=server]').find(".value").find("input").val("localhost"); @@ -18983,7 +18983,7 @@ $form.find('.form-item[rel=rbdsecret]').hide(); $form.find('.form-item[rel=glustervolume]').hide(); - } else if (protocol == "vmfs") { + } else if (protocol == "presetup" && selectedClusterObj.hypervisortype == "VMware") { $form.find('.form-item[rel=server]').css('display', 'inline-block'); $form.find('.form-item[rel=server]').find(".value").find("input").val(""); @@ -19408,7 +19408,7 @@ array1.push("&details[0].user=" + args.data.smbUsername); array1.push("&details[1].password=" + encodeURIComponent(args.data.smbPassword)); array1.push("&details[2].domain=" + args.data.smbDomain); - } else if (args.data.protocol == "PreSetup") { + } else if (args.data.protocol == "PreSetup" && selectedClusterObj.hypervisortype != "VMware") { var path = args.data.path; if (path.substring(0, 1) != "/") path = "/" + path; @@ -19434,12 +19434,12 @@ var rbdid = args.data.rbdid; var rbdsecret = args.data.rbdsecret; url = rbdURL(rbdmonitor, rbdpool, rbdid, rbdsecret); - } else if (args.data.protocol == "vmfs") { + } else if (args.data.protocol == "presetup" && selectedClusterObj.hypervisortype == "VMware") { var path = args.data.vCenterDataCenter; if (path.substring(0, 1) != "/") path = "/" + path; path += "/" + args.data.vCenterDataStore; - url = vmfsURL("dummy", path); + url = presetupURL("dummy", path); } else if (args.data.protocol == "gluster") { var glustervolume = args.data.glustervolume; diff --git a/ui/scripts/zoneWizard.js b/ui/scripts/zoneWizard.js index f4d0e5db521..b7e5d019ce5 100755 --- a/ui/scripts/zoneWizard.js +++ b/ui/scripts/zoneWizard.js @@ -1434,8 +1434,8 @@ description: "nfs" }); items.push({ - id: "vmfs", - description: "vmfs" + id: "presetup", + description: "presetup" }); args.response.success({ data: items @@ -1576,7 +1576,7 @@ $form.find('[rel=rbdsecret]').hide(); $form.find('[rel=glustervolume]').hide(); - } else if (protocol == "PreSetup") { + } else if (protocol == "PreSetup" && selectedClusterObj.hypervisortype != "VMware") { $form.find('[rel=server]').hide(); $form.find('[rel=server]').find(".value").find("input").val("localhost"); @@ -1649,7 +1649,7 @@ $form.find('[rel=rbdsecret]').hide(); $form.find('[rel=glustervolume]').hide(); - } else if (protocol == "vmfs") { + } else if (protocol == "presetup" && selectedClusterObj.hypervisortype == "VMware") { $form.find('[rel=server]').css('display', 'block'); $form.find('[rel=server]').find(".value").find("input").val(""); @@ -4529,7 +4529,7 @@ array1.push("&details[0].user=" + args.data.primaryStorage.smbUsername); array1.push("&details[1].password=" + encodeURIComponent(args.data.primaryStorage.smbPassword)); array1.push("&details[2].domain=" + args.data.primaryStorage.smbDomain); - } else if (args.data.primaryStorage.protocol == "PreSetup") { + } else if (args.data.primaryStorage.protocol == "PreSetup" && selectedClusterObj.hypervisortype != "VMware") { var path = args.data.primaryStorage.path; if (path.substring(0, 1) != "/") path = "/" + path; @@ -4555,12 +4555,12 @@ var rbdid = args.data.primaryStorage.rbdid; var rbdsecret = args.data.primaryStorage.rbdsecret; url = rbdURL(rbdmonitor, rbdpool, rbdid, rbdsecret); - } else if (args.data.primaryStorage.protocol == "vmfs") { + } else if (args.data.primaryStorage.protocol == "presetup" && selectedClusterObj.hypervisortype == "VMware") { var path = args.data.primaryStorage.vCenterDataCenter; if (path.substring(0, 1) != "/") path = "/" + path; path += "/" + args.data.primaryStorage.vCenterDataStore; - url = vmfsURL("dummy", path); + url = presetupURL("dummy", path); } else { var iqn = args.data.primaryStorage.iqn; if (iqn.substring(0, 1) != "/") diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatastoreMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatastoreMO.java index b9eeaeda309..fb98dfd4b9d 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatastoreMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatastoreMO.java @@ -441,4 +441,9 @@ public class DatastoreMO extends BaseMO { return isDatastoreCompatible; } + + public String getDatastoreType() throws Exception { + DatastoreSummary summary = _context.getVimClient().getDynamicProperty(getMor(), "summary"); + return summary.getType(); + } } diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java index e2513ad6fb5..08b5672760a 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java @@ -36,7 +36,6 @@ import javax.xml.datatype.DatatypeConfigurationException; import javax.xml.datatype.DatatypeFactory; import javax.xml.datatype.XMLGregorianCalendar; -import com.vmware.vim25.VirtualDiskType; import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; From 18340e96b101f84f702dbf9cb8921504666770ff Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Mon, 6 Jul 2020 23:58:18 +0530 Subject: [PATCH 020/164] importVsphereStoragePolices improvements --- .../vmware/manager/VmwareManagerImpl.java | 8 +++++ .../vmware/mo/PbmProfileManagerMO.java | 29 +++++++++++++------ 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java index 9b900ad939c..abd5da22595 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java @@ -34,6 +34,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import javax.inject.Inject; import javax.naming.ConfigurationException; @@ -1440,6 +1441,13 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw } } + List allStoragePolicies = vsphereStoragePolicyDao.listAll(); + List finalStorageProfiles = storageProfiles; + List needToMarkRemoved = allStoragePolicies.stream() + .filter(existingPolicy -> finalStorageProfiles.stream() + .anyMatch(storageProfile -> !storageProfile.getProfileId().equals(existingPolicy.getPolicyId()))) + .collect(Collectors.toList()); + List storagePolicies = vsphereStoragePolicyDao.listAll(); return storagePolicies; } diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/PbmProfileManagerMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/PbmProfileManagerMO.java index 557f350cf8e..fc2d7d131d6 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/PbmProfileManagerMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/PbmProfileManagerMO.java @@ -19,7 +19,9 @@ package com.cloud.hypervisor.vmware.mo; import com.cloud.hypervisor.vmware.util.VmwareContext; import com.cloud.utils.exception.CloudRuntimeException; +import com.vmware.pbm.PbmCapabilityProfile; import com.vmware.pbm.PbmProfile; +import com.vmware.pbm.PbmProfileCategoryEnum; import com.vmware.pbm.PbmProfileId; import com.vmware.pbm.PbmProfileResourceType; import com.vmware.pbm.PbmProfileResourceTypeEnum; @@ -29,6 +31,7 @@ import org.apache.log4j.Logger; import java.util.Collections; import java.util.List; +import java.util.stream.Collectors; public class PbmProfileManagerMO extends BaseMO { @@ -57,20 +60,28 @@ public class PbmProfileManagerMO extends BaseMO { public List getStorageProfiles() throws Exception { List profileIds = getStorageProfileIds(); List profiles = _context.getPbmService().pbmRetrieveContent(_mor, profileIds); - return profiles; + + List requirementCategoryProfiles = profiles.stream() + .filter(x -> ((PbmCapabilityProfile)x).getProfileCategory().equals(PbmProfileCategoryEnum.REQUIREMENT)) + .collect(Collectors.toList()); + return requirementCategoryProfiles; } public PbmProfile getStorageProfile(String storageProfileId) throws Exception { List profileIds = getStorageProfileIds(); - for (PbmProfileId profileId : profileIds) { - if (storageProfileId.equals(profileId.getUniqueId())) { - List profile = _context.getPbmService().pbmRetrieveContent(_mor, Collections.singletonList(profileId)); - return profile.get(0); - } + + PbmProfileId profileId = profileIds.stream() + .filter(x -> x.getUniqueId().equals(storageProfileId)) + .findFirst().orElse(null); + + if (profileId == null) { + String errMsg = String.format("Storage profile with id %s not found", storageProfileId); + s_logger.debug(errMsg); + throw new CloudRuntimeException(errMsg); } - String errMsg = String.format("Storage profile with id %s not found", storageProfileId); - s_logger.debug(errMsg); - throw new CloudRuntimeException(errMsg); + + List profile = _context.getPbmService().pbmRetrieveContent(_mor, Collections.singletonList(profileId)); + return profile.get(0); } private PbmProfileResourceType getStorageResourceType() { From 778148ba167e1ee0fe64440e3ca4e54e811c2ee4 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Tue, 7 Jul 2020 01:20:15 +0530 Subject: [PATCH 021/164] Fix importstorage policies to update deleted policies on vcenter --- .../hypervisor/vmware/manager/VmwareManagerImpl.java | 8 ++++++-- .../cloud/hypervisor/vmware/mo/PbmProfileManagerMO.java | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java index abd5da22595..9ac0383bfe8 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java @@ -1444,10 +1444,14 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw List allStoragePolicies = vsphereStoragePolicyDao.listAll(); List finalStorageProfiles = storageProfiles; List needToMarkRemoved = allStoragePolicies.stream() - .filter(existingPolicy -> finalStorageProfiles.stream() - .anyMatch(storageProfile -> !storageProfile.getProfileId().equals(existingPolicy.getPolicyId()))) + .filter(existingPolicy -> !finalStorageProfiles.stream() + .anyMatch(storageProfile -> storageProfile.getProfileId().getUniqueId().equals(existingPolicy.getPolicyId()))) .collect(Collectors.toList()); + for (VsphereStoragePolicyVO storagePolicy : needToMarkRemoved) { + vsphereStoragePolicyDao.remove(storagePolicy.getId()); + } + List storagePolicies = vsphereStoragePolicyDao.listAll(); return storagePolicies; } diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/PbmProfileManagerMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/PbmProfileManagerMO.java index fc2d7d131d6..6858c7b0ffa 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/PbmProfileManagerMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/PbmProfileManagerMO.java @@ -62,7 +62,7 @@ public class PbmProfileManagerMO extends BaseMO { List profiles = _context.getPbmService().pbmRetrieveContent(_mor, profileIds); List requirementCategoryProfiles = profiles.stream() - .filter(x -> ((PbmCapabilityProfile)x).getProfileCategory().equals(PbmProfileCategoryEnum.REQUIREMENT)) + .filter(x -> ((PbmCapabilityProfile)x).getProfileCategory().equals(PbmProfileCategoryEnum.REQUIREMENT.toString())) .collect(Collectors.toList()); return requirementCategoryProfiles; } From 233e665b34b8017e4fba2ca189a506a47644d0aa Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Tue, 7 Jul 2020 02:13:24 +0530 Subject: [PATCH 022/164] Fix response list on importVsphereStoragePoliciescmd --- .../zone/ImportVsphereStoragePoliciesCmd.java | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ImportVsphereStoragePoliciesCmd.java b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ImportVsphereStoragePoliciesCmd.java index cf950003d5b..7e09790a9b5 100644 --- a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ImportVsphereStoragePoliciesCmd.java +++ b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ImportVsphereStoragePoliciesCmd.java @@ -19,7 +19,6 @@ package org.apache.cloudstack.api.command.admin.zone; import com.cloud.dc.DataCenter; import com.cloud.dc.VsphereStoragePolicy; -import com.cloud.event.EventTypes; import com.cloud.exception.ConcurrentOperationException; import com.cloud.exception.InsufficientCapacityException; import com.cloud.exception.NetworkRuleConflictException; @@ -31,7 +30,6 @@ import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; -import org.apache.cloudstack.api.BaseAsyncCmd; import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ServerApiException; @@ -49,7 +47,7 @@ import java.util.List; responseObject = ImportVsphereStoragePoliciesResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, authorized = {RoleType.Admin}) -public class ImportVsphereStoragePoliciesCmd extends BaseAsyncCmd { +public class ImportVsphereStoragePoliciesCmd extends BaseCmd { public static final Logger s_logger = Logger.getLogger(ImportVsphereStoragePoliciesCmd.class.getName()); @@ -62,16 +60,6 @@ public class ImportVsphereStoragePoliciesCmd extends BaseAsyncCmd { description = "ID of the zone") private Long zoneId; - @Override - public String getEventType() { - return EventTypes.EVENT_IMPORT_VCENTER_STORAGE_POLICIES; - } - - @Override - public String getEventDescription() { - return "Importing vSphere Storage Policies"; - } - @Override public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { final DataCenter dataCenter = _resourceService.getZone(getZoneId()); @@ -82,7 +70,7 @@ public class ImportVsphereStoragePoliciesCmd extends BaseAsyncCmd { List storagePolicies = _vmwareDatacenterService.importVsphereStoragePolicies(this); final ListResponse responseList = new ListResponse<>(); final List storagePoliciesResponseList = new ArrayList<>(); - for (VsphereStoragePolicy storagePolicy : storagePolicies) { + for (VsphereStoragePolicy storagePolicy : storagePolicies) { final ImportVsphereStoragePoliciesResponse storagePoliciesResponse = new ImportVsphereStoragePoliciesResponse(); storagePoliciesResponse.setZoneId(dataCenter.getUuid()); storagePoliciesResponse.setId(storagePolicy.getUuid()); From 586fa7eeaff92966c81b0963fd0f038f972e77a9 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Tue, 7 Jul 2020 02:48:25 +0530 Subject: [PATCH 023/164] Added listVsphereStoragePoliciesCmd --- .../admin/offering/CreateDiskOfferingCmd.java | 4 +- ...va => VsphereStoragePoliciesResponse.java} | 2 +- .../cloud/dc/dao/VsphereStoragePolicyDao.java | 4 + .../dc/dao/VsphereStoragePolicyDaoImpl.java | 10 ++ .../vmware/VmwareDatacenterService.java | 19 +-- .../vmware/manager/VmwareManagerImpl.java | 98 +++++++++------- .../zone/ImportVsphereStoragePoliciesCmd.java | 18 ++- .../zone/ListVsphereStoragePoliciesCmd.java | 109 ++++++++++++++++++ tools/apidoc/gen_toc.py | 5 +- 9 files changed, 207 insertions(+), 62 deletions(-) rename api/src/main/java/org/apache/cloudstack/api/response/{ImportVsphereStoragePoliciesResponse.java => VsphereStoragePoliciesResponse.java} (97%) create mode 100644 plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVsphereStoragePoliciesCmd.java diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateDiskOfferingCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateDiskOfferingCmd.java index 4f9eebeb253..6b9ddce75c2 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateDiskOfferingCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateDiskOfferingCmd.java @@ -28,7 +28,7 @@ import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.response.DiskOfferingResponse; import org.apache.cloudstack.api.response.DomainResponse; -import org.apache.cloudstack.api.response.ImportVsphereStoragePoliciesResponse; +import org.apache.cloudstack.api.response.VsphereStoragePoliciesResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; @@ -152,7 +152,7 @@ public class CreateDiskOfferingCmd extends BaseCmd { since = "4.14") private String cacheMode; - @Parameter(name = ApiConstants.STORAGE_POLICY, type = CommandType.UUID, entityType = ImportVsphereStoragePoliciesResponse.class,required = false, description = "Name of the storage policy defined at vCenter, this is applicable only for VMware") + @Parameter(name = ApiConstants.STORAGE_POLICY, type = CommandType.UUID, entityType = VsphereStoragePoliciesResponse.class,required = false, description = "Name of the storage policy defined at vCenter, this is applicable only for VMware") private Long storagePolicy; ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/response/ImportVsphereStoragePoliciesResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/VsphereStoragePoliciesResponse.java similarity index 97% rename from api/src/main/java/org/apache/cloudstack/api/response/ImportVsphereStoragePoliciesResponse.java rename to api/src/main/java/org/apache/cloudstack/api/response/VsphereStoragePoliciesResponse.java index a2227fb8e7f..63c49f14879 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/ImportVsphereStoragePoliciesResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/VsphereStoragePoliciesResponse.java @@ -25,7 +25,7 @@ import org.apache.cloudstack.api.EntityReference; @EntityReference(value = VsphereStoragePolicy.class) -public class ImportVsphereStoragePoliciesResponse extends BaseResponse { +public class VsphereStoragePoliciesResponse extends BaseResponse { @SerializedName(ApiConstants.ID) @Param(description = "the ID of the Storage Policy") diff --git a/engine/schema/src/main/java/com/cloud/dc/dao/VsphereStoragePolicyDao.java b/engine/schema/src/main/java/com/cloud/dc/dao/VsphereStoragePolicyDao.java index e06fd36b6d4..6e79b5e6f6e 100644 --- a/engine/schema/src/main/java/com/cloud/dc/dao/VsphereStoragePolicyDao.java +++ b/engine/schema/src/main/java/com/cloud/dc/dao/VsphereStoragePolicyDao.java @@ -19,8 +19,12 @@ package com.cloud.dc.dao; import com.cloud.dc.VsphereStoragePolicyVO; import com.cloud.utils.db.GenericDao; +import java.util.List; + public interface VsphereStoragePolicyDao extends GenericDao { public VsphereStoragePolicyVO findByPolicyId(Long zoneId, String policyId); + public List findByZoneId(Long zoneId); + } diff --git a/engine/schema/src/main/java/com/cloud/dc/dao/VsphereStoragePolicyDaoImpl.java b/engine/schema/src/main/java/com/cloud/dc/dao/VsphereStoragePolicyDaoImpl.java index fadea5cef25..0cdb6ad7422 100644 --- a/engine/schema/src/main/java/com/cloud/dc/dao/VsphereStoragePolicyDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/dc/dao/VsphereStoragePolicyDaoImpl.java @@ -23,6 +23,8 @@ import com.cloud.utils.db.SearchCriteria; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; +import java.util.List; + @Component public class VsphereStoragePolicyDaoImpl extends GenericDaoBase implements VsphereStoragePolicyDao { @@ -51,4 +53,12 @@ public class VsphereStoragePolicyDaoImpl extends GenericDaoBase findByZoneId(Long zoneId) { + SearchCriteria sc = zoneSearch.create(); + sc.setParameters("zoneId", zoneId); + + return listBy(sc); + } } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenterService.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenterService.java index 6e69009b62f..2e3f98d795a 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenterService.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenterService.java @@ -17,19 +17,19 @@ package com.cloud.hypervisor.vmware; -import java.util.List; - import com.cloud.dc.VsphereStoragePolicy; -import org.apache.cloudstack.api.command.admin.zone.AddVmwareDcCmd; -import org.apache.cloudstack.api.command.admin.zone.ImportVsphereStoragePoliciesCmd; -import org.apache.cloudstack.api.command.admin.zone.ListVmwareDcsCmd; -import org.apache.cloudstack.api.command.admin.zone.RemoveVmwareDcCmd; -import org.apache.cloudstack.api.command.admin.zone.UpdateVmwareDcCmd; - import com.cloud.exception.DiscoveryException; import com.cloud.exception.ResourceInUseException; import com.cloud.utils.component.PluggableService; import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.api.command.admin.zone.AddVmwareDcCmd; +import org.apache.cloudstack.api.command.admin.zone.ImportVsphereStoragePoliciesCmd; +import org.apache.cloudstack.api.command.admin.zone.ListVmwareDcsCmd; +import org.apache.cloudstack.api.command.admin.zone.ListVsphereStoragePoliciesCmd; +import org.apache.cloudstack.api.command.admin.zone.RemoveVmwareDcCmd; +import org.apache.cloudstack.api.command.admin.zone.UpdateVmwareDcCmd; + +import java.util.List; public interface VmwareDatacenterService extends PluggableService { @@ -42,4 +42,7 @@ public interface VmwareDatacenterService extends PluggableService { List listVmwareDatacenters(ListVmwareDcsCmd cmd) throws IllegalArgumentException, CloudRuntimeException; List importVsphereStoragePolicies(ImportVsphereStoragePoliciesCmd cmd); + + List listVsphereStoragePolicies(ListVsphereStoragePoliciesCmd cmd); + } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java index 9ac0383bfe8..e10e88c19dd 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java @@ -16,50 +16,6 @@ // under the License. package com.cloud.hypervisor.vmware.manager; -import java.io.File; -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; -import java.rmi.RemoteException; -import java.time.Duration; -import java.time.Instant; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.UUID; -import java.util.concurrent.Executors; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - -import javax.inject.Inject; -import javax.naming.ConfigurationException; - -import com.cloud.dc.VsphereStoragePolicy; -import com.cloud.dc.VsphereStoragePolicyVO; -import com.cloud.dc.dao.VsphereStoragePolicyDao; -import com.cloud.hypervisor.vmware.mo.PbmProfileManagerMO; -import com.vmware.pbm.PbmProfile; -import org.apache.cloudstack.api.command.admin.zone.AddVmwareDcCmd; -import org.apache.cloudstack.api.command.admin.zone.ImportVsphereStoragePoliciesCmd; -import org.apache.cloudstack.api.command.admin.zone.ListVmwareDcsCmd; -import org.apache.cloudstack.api.command.admin.zone.RemoveVmwareDcCmd; -import org.apache.cloudstack.api.command.admin.zone.UpdateVmwareDcCmd; -import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; -import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; -import org.apache.cloudstack.framework.config.ConfigKey; -import org.apache.cloudstack.framework.config.Configurable; -import org.apache.cloudstack.framework.config.dao.ConfigurationDao; -import org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl; -import org.apache.cloudstack.management.ManagementServerHost; -import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; -import org.apache.cloudstack.utils.identity.ManagementServerNode; -import org.apache.log4j.Logger; - import com.amazonaws.util.CollectionUtils; import com.cloud.agent.AgentManager; import com.cloud.agent.Listener; @@ -77,9 +33,12 @@ import com.cloud.dc.ClusterDetailsDao; import com.cloud.dc.ClusterVO; import com.cloud.dc.ClusterVSMMapVO; import com.cloud.dc.DataCenterVO; +import com.cloud.dc.VsphereStoragePolicy; +import com.cloud.dc.VsphereStoragePolicyVO; import com.cloud.dc.dao.ClusterDao; import com.cloud.dc.dao.ClusterVSMMapDao; import com.cloud.dc.dao.DataCenterDao; +import com.cloud.dc.dao.VsphereStoragePolicyDao; import com.cloud.event.ActionEvent; import com.cloud.event.EventTypes; import com.cloud.exception.DiscoveredWithErrorException; @@ -109,6 +68,7 @@ import com.cloud.hypervisor.vmware.mo.DiskControllerType; import com.cloud.hypervisor.vmware.mo.HostFirewallSystemMO; import com.cloud.hypervisor.vmware.mo.HostMO; import com.cloud.hypervisor.vmware.mo.HypervisorHostHelper; +import com.cloud.hypervisor.vmware.mo.PbmProfileManagerMO; import com.cloud.hypervisor.vmware.mo.VirtualEthernetCardType; import com.cloud.hypervisor.vmware.mo.VirtualSwitchType; import com.cloud.hypervisor.vmware.mo.VmwareHostType; @@ -149,8 +109,48 @@ import com.cloud.vm.DomainRouterVO; import com.cloud.vm.dao.UserVmCloneSettingDao; import com.cloud.vm.dao.VMInstanceDao; import com.google.common.base.Strings; +import com.vmware.pbm.PbmProfile; import com.vmware.vim25.AboutInfo; import com.vmware.vim25.ManagedObjectReference; +import org.apache.cloudstack.api.command.admin.zone.AddVmwareDcCmd; +import org.apache.cloudstack.api.command.admin.zone.ImportVsphereStoragePoliciesCmd; +import org.apache.cloudstack.api.command.admin.zone.ListVmwareDcsCmd; +import org.apache.cloudstack.api.command.admin.zone.ListVsphereStoragePoliciesCmd; +import org.apache.cloudstack.api.command.admin.zone.RemoveVmwareDcCmd; +import org.apache.cloudstack.api.command.admin.zone.UpdateVmwareDcCmd; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl; +import org.apache.cloudstack.management.ManagementServerHost; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.utils.identity.ManagementServerNode; +import org.apache.log4j.Logger; + +import javax.inject.Inject; +import javax.naming.ConfigurationException; +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.rmi.RemoteException; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; public class VmwareManagerImpl extends ManagerBase implements VmwareManager, VmwareStorageMount, Listener, VmwareDatacenterService, Configurable { private static final Logger s_logger = Logger.getLogger(VmwareManagerImpl.class); @@ -1056,6 +1056,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw cmdList.add(RemoveVmwareDcCmd.class); cmdList.add(ListVmwareDcsCmd.class); cmdList.add(ImportVsphereStoragePoliciesCmd.class); + cmdList.add(ListVsphereStoragePoliciesCmd.class); return cmdList; } @@ -1456,6 +1457,15 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw return storagePolicies; } + @Override + public List listVsphereStoragePolicies(ListVsphereStoragePoliciesCmd cmd) { + List storagePolicies = vsphereStoragePolicyDao.findByZoneId(cmd.getZoneId()); + if (storagePolicies != null) { + return new ArrayList<>(storagePolicies); + } + return Collections.emptyList(); + } + @Override public boolean hasNexusVSM(Long clusterId) { ClusterVSMMapVO vsmMapVo = null; diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ImportVsphereStoragePoliciesCmd.java b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ImportVsphereStoragePoliciesCmd.java index 7e09790a9b5..ccb77de71f8 100644 --- a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ImportVsphereStoragePoliciesCmd.java +++ b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ImportVsphereStoragePoliciesCmd.java @@ -33,7 +33,7 @@ import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ServerApiException; -import org.apache.cloudstack.api.response.ImportVsphereStoragePoliciesResponse; +import org.apache.cloudstack.api.response.VsphereStoragePoliciesResponse; import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.cloudstack.context.CallContext; @@ -44,7 +44,7 @@ import java.util.ArrayList; import java.util.List; @APICommand(name = ImportVsphereStoragePoliciesCmd.APINAME, description = "Import vSphere storage policies", - responseObject = ImportVsphereStoragePoliciesResponse.class, + responseObject = VsphereStoragePoliciesResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, authorized = {RoleType.Admin}) public class ImportVsphereStoragePoliciesCmd extends BaseCmd { @@ -56,10 +56,18 @@ public class ImportVsphereStoragePoliciesCmd extends BaseCmd { @Inject public VmwareDatacenterService _vmwareDatacenterService; + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.UUID, entityType = ZoneResponse.class, description = "ID of the zone") private Long zoneId; + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + @Override public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { final DataCenter dataCenter = _resourceService.getZone(getZoneId()); @@ -68,10 +76,10 @@ public class ImportVsphereStoragePoliciesCmd extends BaseCmd { } List storagePolicies = _vmwareDatacenterService.importVsphereStoragePolicies(this); - final ListResponse responseList = new ListResponse<>(); - final List storagePoliciesResponseList = new ArrayList<>(); + final ListResponse responseList = new ListResponse<>(); + final List storagePoliciesResponseList = new ArrayList<>(); for (VsphereStoragePolicy storagePolicy : storagePolicies) { - final ImportVsphereStoragePoliciesResponse storagePoliciesResponse = new ImportVsphereStoragePoliciesResponse(); + final VsphereStoragePoliciesResponse storagePoliciesResponse = new VsphereStoragePoliciesResponse(); storagePoliciesResponse.setZoneId(dataCenter.getUuid()); storagePoliciesResponse.setId(storagePolicy.getUuid()); storagePoliciesResponse.setName(storagePolicy.getName()); diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVsphereStoragePoliciesCmd.java b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVsphereStoragePoliciesCmd.java new file mode 100644 index 00000000000..26a34a874e2 --- /dev/null +++ b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVsphereStoragePoliciesCmd.java @@ -0,0 +1,109 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.admin.zone; + +import com.cloud.dc.DataCenter; +import com.cloud.dc.VsphereStoragePolicy; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.hypervisor.vmware.VmwareDatacenterService; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.VsphereStoragePoliciesResponse; +import org.apache.cloudstack.api.response.ZoneResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.log4j.Logger; + +import javax.inject.Inject; +import java.util.ArrayList; +import java.util.List; + +@APICommand(name = ListVsphereStoragePoliciesCmd.APINAME, description = "List vSphere storage policies", + responseObject = VsphereStoragePoliciesResponse.class, + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, + authorized = {RoleType.Admin}) +public class ListVsphereStoragePoliciesCmd extends BaseCmd { + + public static final Logger s_logger = Logger.getLogger(ListVsphereStoragePoliciesCmd.class.getName()); + + public static final String APINAME = "listVsphereStoragePolicies"; + + @Inject + public VmwareDatacenterService _vmwareDatacenterService; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.UUID, entityType = ZoneResponse.class, + description = "ID of the zone") + private Long zoneId; + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { + final DataCenter dataCenter = _resourceService.getZone(getZoneId()); + if (dataCenter == null) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Unable to find zone by ID: " + getZoneId()); + } + + List storagePolicies = _vmwareDatacenterService.listVsphereStoragePolicies(this); + final ListResponse responseList = new ListResponse<>(); + final List storagePoliciesResponseList = new ArrayList<>(); + for (VsphereStoragePolicy storagePolicy : storagePolicies) { + final VsphereStoragePoliciesResponse storagePoliciesResponse = new VsphereStoragePoliciesResponse(); + storagePoliciesResponse.setZoneId(dataCenter.getUuid()); + storagePoliciesResponse.setId(storagePolicy.getUuid()); + storagePoliciesResponse.setName(storagePolicy.getName()); + storagePoliciesResponse.setPolicyId(storagePolicy.getPolicyId()); + storagePoliciesResponse.setDescription(storagePolicy.getDescription()); + storagePoliciesResponse.setObjectName("StoragePolicy"); + + storagePoliciesResponseList.add(storagePoliciesResponse); + } + responseList.setResponses(storagePoliciesResponseList); + responseList.setResponseName(getCommandName()); + setResponseObject(responseList); + } + + @Override + public String getCommandName() { + return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX; + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccountId(); + } + + public Long getZoneId() { + return zoneId; + } +} diff --git a/tools/apidoc/gen_toc.py b/tools/apidoc/gen_toc.py index 73b8e2e871e..0b0702ac86c 100644 --- a/tools/apidoc/gen_toc.py +++ b/tools/apidoc/gen_toc.py @@ -197,8 +197,9 @@ known_categories = { 'KubernetesCluster': 'Kubernetes Service', 'UnmanagedInstance': 'Virtual Machine', 'Rolling': 'Rolling Maintenance', - 'importVsphereStoragePolicies' : 'vSphere storage policies' - } + 'importVsphereStoragePolicies' : 'vSphere storage policies', + 'listVsphereStoragePolicies' : 'vSphere storage policies' +} categories = {} From c45b83a158c48972a24c456b888d15cc9841c7d1 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Tue, 7 Jul 2020 22:36:57 +0530 Subject: [PATCH 024/164] Logging variable name change --- .../admin/zone/ImportVsphereStoragePoliciesCmd.java | 2 +- .../command/admin/zone/ListVsphereStoragePoliciesCmd.java | 2 +- .../cloud/hypervisor/vmware/mo/PbmPlacementSolverMO.java | 2 +- .../cloud/hypervisor/vmware/mo/PbmProfileManagerMO.java | 8 ++++---- .../vmware/mo/VirtualStorageObjectManagerMO.java | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ImportVsphereStoragePoliciesCmd.java b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ImportVsphereStoragePoliciesCmd.java index ccb77de71f8..ea5bacfb1f4 100644 --- a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ImportVsphereStoragePoliciesCmd.java +++ b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ImportVsphereStoragePoliciesCmd.java @@ -49,7 +49,7 @@ import java.util.List; authorized = {RoleType.Admin}) public class ImportVsphereStoragePoliciesCmd extends BaseCmd { - public static final Logger s_logger = Logger.getLogger(ImportVsphereStoragePoliciesCmd.class.getName()); + public static final Logger LOGGER = Logger.getLogger(ImportVsphereStoragePoliciesCmd.class.getName()); public static final String APINAME = "importVsphereStoragePolicies"; diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVsphereStoragePoliciesCmd.java b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVsphereStoragePoliciesCmd.java index 26a34a874e2..90e8e880548 100644 --- a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVsphereStoragePoliciesCmd.java +++ b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVsphereStoragePoliciesCmd.java @@ -48,7 +48,7 @@ import java.util.List; authorized = {RoleType.Admin}) public class ListVsphereStoragePoliciesCmd extends BaseCmd { - public static final Logger s_logger = Logger.getLogger(ListVsphereStoragePoliciesCmd.class.getName()); + public static final Logger LOGGER = Logger.getLogger(ListVsphereStoragePoliciesCmd.class.getName()); public static final String APINAME = "listVsphereStoragePolicies"; diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/PbmPlacementSolverMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/PbmPlacementSolverMO.java index 9735f424d88..3eb909fc31b 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/PbmPlacementSolverMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/PbmPlacementSolverMO.java @@ -30,7 +30,7 @@ import java.util.List; public class PbmPlacementSolverMO extends BaseMO { - private static final Logger s_logger = Logger.getLogger(PbmPlacementSolverMO.class); + private static final Logger LOGGER = Logger.getLogger(PbmPlacementSolverMO.class); public PbmPlacementSolverMO (VmwareContext context) { super(context, context.getPbmServiceContent().getPlacementSolver()); diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/PbmProfileManagerMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/PbmProfileManagerMO.java index 6858c7b0ffa..a4142ecde8a 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/PbmProfileManagerMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/PbmProfileManagerMO.java @@ -35,7 +35,7 @@ import java.util.stream.Collectors; public class PbmProfileManagerMO extends BaseMO { - private static final Logger s_logger = Logger.getLogger(PbmProfileManagerMO.class); + private static final Logger LOGGER = Logger.getLogger(PbmProfileManagerMO.class); public PbmProfileManagerMO (VmwareContext context) { super(context, context.getPbmServiceContent().getProfileManager()); @@ -50,8 +50,8 @@ public class PbmProfileManagerMO extends BaseMO { } public List getStorageProfileIds() throws Exception { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Querying vCenter " + _context.getServerAddress() + " for profiles"); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Querying vCenter " + _context.getServerAddress() + " for profiles"); } List profileIds = _context.getPbmService().pbmQueryProfile(_mor, getStorageResourceType(), null); return profileIds; @@ -76,7 +76,7 @@ public class PbmProfileManagerMO extends BaseMO { if (profileId == null) { String errMsg = String.format("Storage profile with id %s not found", storageProfileId); - s_logger.debug(errMsg); + LOGGER.debug(errMsg); throw new CloudRuntimeException(errMsg); } diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualStorageObjectManagerMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualStorageObjectManagerMO.java index 7924dfda646..d5f4eb3af06 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualStorageObjectManagerMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualStorageObjectManagerMO.java @@ -30,7 +30,7 @@ import com.cloud.hypervisor.vmware.util.VmwareContext; public class VirtualStorageObjectManagerMO extends BaseMO { @SuppressWarnings("unused") - private static final Logger s_logger = Logger.getLogger(VirtualStorageObjectManagerMO.class); + private static final Logger LOGGER = Logger.getLogger(VirtualStorageObjectManagerMO.class); public VirtualStorageObjectManagerMO(VmwareContext context) { super(context, context.getServiceContent().getVStorageObjectManager()); @@ -86,7 +86,7 @@ public class VirtualStorageObjectManagerMO extends BaseMO { vStorageObject = (VStorageObject)taskInfo.getResult(); } else { - s_logger.error("VMware CreateDisk_Task failed due to " + TaskMO.getTaskFailureInfo(_context, morTask)); + LOGGER.error("VMware CreateDisk_Task failed due to " + TaskMO.getTaskFailureInfo(_context, morTask)); } return vStorageObject; From 48786b2d312550e7b724783b9ac2f3ca5c58e186 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Mon, 13 Jul 2020 22:14:01 +0530 Subject: [PATCH 025/164] DataStore Clusters addition as a storage pool --- .../main/java/com/cloud/storage/Storage.java | 7 +- .../java/com/cloud/storage/StorageTest.java | 2 + .../provider/DefaultHostListener.java | 32 ++--- .../vmware/resource/VmwareResource.java | 123 +++++++++------- .../resource/VmwareStorageProcessor.java | 2 +- ...oudStackPrimaryDataStoreLifeCycleImpl.java | 56 ++++---- .../hypervisor/vmware/mo/DatastoreMO.java | 21 +-- .../vmware/mo/HostDatastoreSystemMO.java | 135 +++++++++++++++++- .../cloud/hypervisor/vmware/mo/HostMO.java | 16 ++- .../vmware/mo/HypervisorHostHelper.java | 16 ++- .../hypervisor/vmware/util/VmwareContext.java | 18 +++ 11 files changed, 305 insertions(+), 123 deletions(-) diff --git a/api/src/main/java/com/cloud/storage/Storage.java b/api/src/main/java/com/cloud/storage/Storage.java index 82bc5f6d4e5..7a229b67674 100644 --- a/api/src/main/java/com/cloud/storage/Storage.java +++ b/api/src/main/java/com/cloud/storage/Storage.java @@ -16,11 +16,11 @@ // under the License. package com.cloud.storage; +import org.apache.commons.lang.NotImplementedException; + import java.util.ArrayList; import java.util.List; -import org.apache.commons.lang.NotImplementedException; - public class Storage { public static enum ImageFormat { QCOW2(true, true, false, "qcow2"), @@ -135,7 +135,8 @@ public class Storage { OCFS2(true, false), SMB(true, false), Gluster(true, false), - ManagedNFS(true, false); + ManagedNFS(true, false), + DatastoreCluster(true, true); // for VMware, to abstract pool of clusters private final boolean shared; private final boolean overprovisioning; diff --git a/api/src/test/java/com/cloud/storage/StorageTest.java b/api/src/test/java/com/cloud/storage/StorageTest.java index 332a8060d08..c57130f5418 100644 --- a/api/src/test/java/com/cloud/storage/StorageTest.java +++ b/api/src/test/java/com/cloud/storage/StorageTest.java @@ -45,6 +45,7 @@ public class StorageTest { Assert.assertTrue(StoragePoolType.SMB.isShared()); Assert.assertTrue(StoragePoolType.Gluster.isShared()); Assert.assertTrue(StoragePoolType.ManagedNFS.isShared()); + Assert.assertTrue(StoragePoolType.DatastoreCluster.isShared()); } @Test @@ -66,5 +67,6 @@ public class StorageTest { Assert.assertFalse(StoragePoolType.SMB.supportsOverProvisioning()); Assert.assertFalse(StoragePoolType.Gluster.supportsOverProvisioning()); Assert.assertFalse(StoragePoolType.ManagedNFS.supportsOverProvisioning()); + Assert.assertFalse(StoragePoolType.DatastoreCluster.supportsOverProvisioning()); } } diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java index 1b9682a2fa3..3ba45191147 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java @@ -18,20 +18,6 @@ */ package org.apache.cloudstack.storage.datastore.provider; -import java.util.List; - -import javax.inject.Inject; - -import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailVO; -import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; -import org.apache.commons.lang.StringUtils; -import org.apache.log4j.Logger; - -import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; -import org.apache.cloudstack.engine.subsystem.api.storage.HypervisorHostListener; -import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; - import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; import com.cloud.agent.api.ModifyStoragePoolAnswer; @@ -43,6 +29,17 @@ import com.cloud.storage.StoragePool; import com.cloud.storage.StoragePoolHostVO; import com.cloud.storage.dao.StoragePoolHostDao; import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.HypervisorHostListener; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailVO; +import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.commons.lang.StringUtils; +import org.apache.log4j.Logger; + +import javax.inject.Inject; +import java.util.List; public class DefaultHostListener implements HypervisorHostListener { private static final Logger s_logger = Logger.getLogger(DefaultHostListener.class); @@ -108,8 +105,11 @@ public class DefaultHostListener implements HypervisorHostListener { poolVO.setUsedBytes(mspAnswer.getPoolInfo().getCapacityBytes() - mspAnswer.getPoolInfo().getAvailableBytes()); poolVO.setCapacityBytes(mspAnswer.getPoolInfo().getCapacityBytes()); if(StringUtils.isNotEmpty(mspAnswer.getPoolType())) { - StoragePoolDetailVO storagePoolDetailVO = new StoragePoolDetailVO(poolId, "pool_type", mspAnswer.getPoolType(), false); - storagePoolDetailsDao.persist(storagePoolDetailVO); + StoragePoolDetailVO poolType = storagePoolDetailsDao.findDetail(poolId, "pool_type"); + if (poolType == null) { + StoragePoolDetailVO storagePoolDetailVO = new StoragePoolDetailVO(poolId, "pool_type", mspAnswer.getPoolType(), false); + storagePoolDetailsDao.persist(storagePoolDetailVO); + } } primaryStoreDao.update(pool.getId(), poolVO); diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 1230fa8fe1a..ffeebc7c5f3 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -16,50 +16,9 @@ // under the License. package com.cloud.hypervisor.vmware.resource; -import java.io.File; -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.net.ConnectException; -import java.net.InetSocketAddress; -import java.net.URI; -import java.net.URL; -import java.nio.channels.SocketChannel; -import java.rmi.RemoteException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.Date; -import java.util.EnumMap; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.Set; -import java.util.TimeZone; -import java.util.UUID; - -import javax.naming.ConfigurationException; -import javax.xml.datatype.XMLGregorianCalendar; - import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.storage.command.CopyCommand; -import org.apache.cloudstack.storage.command.StorageSubSystemCommand; import org.apache.cloudstack.storage.configdrive.ConfigDrive; -import org.apache.cloudstack.storage.resource.NfsSecondaryStorageResource; -import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; import org.apache.cloudstack.storage.to.TemplateObjectTO; -import org.apache.cloudstack.storage.to.VolumeObjectTO; -import org.apache.cloudstack.utils.volume.VirtualMachineDiskInfo; -import org.apache.cloudstack.vm.UnmanagedInstanceTO; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang.math.NumberUtils; -import org.apache.log4j.Logger; -import org.apache.log4j.NDC; -import org.joda.time.Duration; import com.cloud.agent.IAgentControl; import com.cloud.agent.api.Answer; @@ -297,6 +256,7 @@ import com.vmware.vim25.PerfMetricIntSeries; import com.vmware.vim25.PerfMetricSeries; import com.vmware.vim25.PerfQuerySpec; import com.vmware.vim25.RuntimeFaultFaultMsg; +import com.vmware.vim25.StoragePodSummary; import com.vmware.vim25.ToolsUnavailableFaultMsg; import com.vmware.vim25.VAppOvfSectionInfo; import com.vmware.vim25.VAppOvfSectionSpec; @@ -340,6 +300,46 @@ import com.vmware.vim25.VmConfigSpec; import com.vmware.vim25.VmfsDatastoreInfo; import com.vmware.vim25.VmwareDistributedVirtualSwitchPvlanSpec; import com.vmware.vim25.VmwareDistributedVirtualSwitchVlanIdSpec; +import org.apache.cloudstack.storage.command.CopyCommand; +import org.apache.cloudstack.storage.command.StorageSubSystemCommand; +import org.apache.cloudstack.storage.resource.NfsSecondaryStorageResource; +import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; +import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.apache.cloudstack.utils.volume.VirtualMachineDiskInfo; +import org.apache.cloudstack.vm.UnmanagedInstanceTO; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.math.NumberUtils; +import org.apache.log4j.Logger; +import org.apache.log4j.NDC; +import org.joda.time.Duration; + +import javax.naming.ConfigurationException; +import javax.xml.datatype.XMLGregorianCalendar; +import java.io.File; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.ConnectException; +import java.net.InetSocketAddress; +import java.net.URI; +import java.net.URL; +import java.nio.channels.SocketChannel; +import java.rmi.RemoteException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.Date; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.TimeZone; +import java.util.UUID; import static com.cloud.utils.HumanReadableJson.getHumanReadableBytesJson; import static com.cloud.utils.NumbersUtil.toHumanReadableSize; @@ -4901,7 +4901,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa VmwareHypervisorHost hyperHost = getHyperHost(getServiceContext()); StorageFilerTO pool = cmd.getPool(); - if (pool.getType() != StoragePoolType.NetworkFilesystem && pool.getType() != StoragePoolType.VMFS && pool.getType() != StoragePoolType.PreSetup) { + if (pool.getType() != StoragePoolType.NetworkFilesystem && pool.getType() != StoragePoolType.VMFS && pool.getType() != StoragePoolType.PreSetup && pool.getType() != StoragePoolType.DatastoreCluster) { throw new Exception("Unsupported storage pool type " + pool.getType()); } @@ -4914,16 +4914,24 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa assert (morDatastore != null); DatastoreMO dsMo = new DatastoreMO(getServiceContext(), morDatastore); - HypervisorHostHelper.createBaseFolderInDatastore(dsMo, hyperHost); + HypervisorHostHelper.createBaseFolder(dsMo, hyperHost, pool.getType()); - DatastoreSummary summary = dsMo.getSummary(); - long capacity = summary.getCapacity(); - long available = summary.getFreeSpace(); + long capacity = 0; + long available = 0; + if (pool.getType() == StoragePoolType.DatastoreCluster) { + StoragePodSummary summary = dsMo.getDatastoreClusterSummary(); + capacity = summary.getCapacity(); + available = summary.getFreeSpace(); + } else { + DatastoreSummary summary = dsMo.getDatastoreSummary(); + capacity = summary.getCapacity(); + available = summary.getFreeSpace(); + } Map tInfo = new HashMap<>(); ModifyStoragePoolAnswer answer = new ModifyStoragePoolAnswer(cmd, capacity, available, tInfo); - if (cmd.getAdd() && (pool.getType() == StoragePoolType.VMFS || pool.getType() == StoragePoolType.PreSetup)) { + if (cmd.getAdd() && (pool.getType() == StoragePoolType.VMFS || pool.getType() == StoragePoolType.PreSetup) && pool.getType() != StoragePoolType.DatastoreCluster) { answer.setPoolType(dsMo.getDatastoreType()); answer.setLocalDatastoreName(morDatastore.getValue()); } @@ -5305,11 +5313,18 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa if (morDs != null) { DatastoreMO datastoreMo = new DatastoreMO(context, morDs); - DatastoreSummary summary = datastoreMo.getSummary(); - assert (summary != null); + long capacity = 0; + long free = 0; + if (cmd.getPooltype() == StoragePoolType.DatastoreCluster) { + StoragePodSummary summary = datastoreMo.getDatastoreClusterSummary(); + capacity = summary.getCapacity(); + free = summary.getFreeSpace(); + } else { + DatastoreSummary summary = datastoreMo.getDatastoreSummary(); + capacity = summary.getCapacity(); + free = summary.getFreeSpace(); + } - long capacity = summary.getCapacity(); - long free = summary.getFreeSpace(); long used = capacity - free; if (s_logger.isDebugEnabled()) { @@ -5317,7 +5332,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa + ", capacity: " + toHumanReadableSize(capacity) + ", free: " + toHumanReadableSize(free) + ", used: " + toHumanReadableSize(used)); } - if (summary.getCapacity() <= 0) { + if (capacity <= 0) { s_logger.warn("Something is wrong with vSphere NFS datastore, rebooting ESX(ESXi) host should help"); } @@ -5839,7 +5854,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa dsMo.setCustomFieldValue(CustomFieldConstants.CLOUD_UUID, poolUuid); } - DatastoreSummary dsSummary = dsMo.getSummary(); + DatastoreSummary dsSummary = dsMo.getDatastoreSummary(); String address = hostMo.getHostName(); StoragePoolInfo pInfo = new StoragePoolInfo(poolUuid, address, dsMo.getMor().getValue(), "", StoragePoolType.VMFS, dsSummary.getCapacity(), dsSummary.getFreeSpace()); @@ -6555,6 +6570,8 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa CustomFieldsManagerMO cfmMo = new CustomFieldsManagerMO(context, context.getServiceContent().getCustomFieldsManager()); cfmMo.ensureCustomFieldDef("Datastore", CustomFieldConstants.CLOUD_UUID); + cfmMo.ensureCustomFieldDef("StoragePod", CustomFieldConstants.CLOUD_UUID); + if (_publicTrafficInfo != null && _publicTrafficInfo.getVirtualSwitchType() != VirtualSwitchType.StandardVirtualSwitch || _guestTrafficInfo != null && _guestTrafficInfo.getVirtualSwitchType() != VirtualSwitchType.StandardVirtualSwitch) { cfmMo.ensureCustomFieldDef("DistributedVirtualPortgroup", CustomFieldConstants.CLOUD_GC_DVP); diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java index 9cd9b6e9ea3..22d0a92aea6 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -2962,7 +2962,7 @@ public class VmwareStorageProcessor implements StorageProcessor { throw new Exception("Unable to create a dummy VM for volume creation"); } - Long volumeSizeToUse = volumeSize < dsMo.getSummary().getFreeSpace() ? volumeSize : dsMo.getSummary().getFreeSpace(); + Long volumeSizeToUse = volumeSize < dsMo.getDatastoreSummary().getFreeSpace() ? volumeSize : dsMo.getDatastoreSummary().getFreeSpace(); vmMo.createDisk(vmdkDatastorePath, getMBsFromBytes(volumeSizeToUse), dsMo.getMor(), vmMo.getScsiDeviceControllerKey()); vmMo.detachDisk(vmdkDatastorePath, false); diff --git a/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java b/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java index 84b7bfdf9af..955af589c19 100644 --- a/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java +++ b/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java @@ -18,32 +18,6 @@ */ package org.apache.cloudstack.storage.datastore.lifecycle; -import java.io.UnsupportedEncodingException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URLDecoder; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.UUID; - -import javax.inject.Inject; - -import org.apache.commons.lang.StringUtils; -import org.apache.log4j.Logger; - -import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope; -import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; -import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; -import org.apache.cloudstack.engine.subsystem.api.storage.HostScope; -import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreInfo; -import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreLifeCycle; -import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreParameters; -import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; -import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; -import org.apache.cloudstack.storage.volume.datastore.PrimaryDataStoreHelper; - import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; import com.cloud.agent.api.CreateStoragePoolCommand; @@ -78,6 +52,29 @@ import com.cloud.vm.dao.DomainRouterDao; import com.cloud.vm.dao.SecondaryStorageVmDao; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.VMInstanceDao; +import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.HostScope; +import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreLifeCycle; +import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreParameters; +import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.storage.volume.datastore.PrimaryDataStoreHelper; +import org.apache.commons.lang.StringUtils; +import org.apache.log4j.Logger; + +import javax.inject.Inject; +import java.io.UnsupportedEncodingException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URLDecoder; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.UUID; public class CloudStackPrimaryDataStoreLifeCycleImpl implements PrimaryDataStoreLifeCycle { private static final Logger s_logger = Logger.getLogger(CloudStackPrimaryDataStoreLifeCycleImpl.class); @@ -256,6 +253,11 @@ public class CloudStackPrimaryDataStoreLifeCycleImpl implements PrimaryDataStore parameters.setHost(storageHost); parameters.setPort(0); parameters.setPath(hostPath); + } else if (scheme.equalsIgnoreCase("DatastoreCluster")) { + parameters.setType(StoragePoolType.DatastoreCluster); + parameters.setHost(storageHost); + parameters.setPort(0); + parameters.setPath(hostPath); } else if (scheme.equalsIgnoreCase("iscsi")) { String[] tokens = hostPath.split("/"); int lun = NumbersUtil.parseInt(tokens[tokens.length - 1], -1); @@ -360,7 +362,7 @@ public class CloudStackPrimaryDataStoreLifeCycleImpl implements PrimaryDataStore if (pool.getPoolType() != StoragePoolType.NetworkFilesystem && pool.getPoolType() != StoragePoolType.Filesystem && pool.getPoolType() != StoragePoolType.IscsiLUN && pool.getPoolType() != StoragePoolType.Iscsi && pool.getPoolType() != StoragePoolType.VMFS && - pool.getPoolType() != StoragePoolType.SharedMountPoint && pool.getPoolType() != StoragePoolType.PreSetup && pool.getPoolType() != StoragePoolType.OCFS2 && + pool.getPoolType() != StoragePoolType.SharedMountPoint && pool.getPoolType() != StoragePoolType.PreSetup && pool.getPoolType() != StoragePoolType.DatastoreCluster && pool.getPoolType() != StoragePoolType.OCFS2 && pool.getPoolType() != StoragePoolType.RBD && pool.getPoolType() != StoragePoolType.CLVM && pool.getPoolType() != StoragePoolType.SMB && pool.getPoolType() != StoragePoolType.Gluster) { s_logger.warn(" Doesn't support storage pool type " + pool.getPoolType()); diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatastoreMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatastoreMO.java index fb98dfd4b9d..133a412d750 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatastoreMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatastoreMO.java @@ -16,12 +16,10 @@ // under the License. package com.cloud.hypervisor.vmware.mo; -import java.util.ArrayList; -import java.util.List; - +import com.cloud.exception.CloudException; +import com.cloud.hypervisor.vmware.util.VmwareContext; +import com.cloud.utils.Pair; import com.vmware.pbm.PbmProfile; -import org.apache.log4j.Logger; - import com.vmware.vim25.DatastoreHostMount; import com.vmware.vim25.DatastoreSummary; import com.vmware.vim25.FileInfo; @@ -35,11 +33,12 @@ import com.vmware.vim25.ObjectSpec; import com.vmware.vim25.PropertyFilterSpec; import com.vmware.vim25.PropertySpec; import com.vmware.vim25.SelectionSpec; +import com.vmware.vim25.StoragePodSummary; import com.vmware.vim25.TraversalSpec; +import org.apache.log4j.Logger; -import com.cloud.exception.CloudException; -import com.cloud.hypervisor.vmware.util.VmwareContext; -import com.cloud.utils.Pair; +import java.util.ArrayList; +import java.util.List; import static com.cloud.utils.NumbersUtil.toHumanReadableSize; @@ -65,10 +64,14 @@ public class DatastoreMO extends BaseMO { return _name; } - public DatastoreSummary getSummary() throws Exception { + public DatastoreSummary getDatastoreSummary() throws Exception { return (DatastoreSummary)_context.getVimClient().getDynamicProperty(_mor, "summary"); } + public StoragePodSummary getDatastoreClusterSummary() throws Exception { + return (StoragePodSummary)_context.getVimClient().getDynamicProperty(_mor, "summary"); + } + public HostDatastoreBrowserMO getHostDatastoreBrowserMO() throws Exception { return new HostDatastoreBrowserMO(_context, (ManagedObjectReference)_context.getVimClient().getDynamicProperty(_mor, "browser")); } diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HostDatastoreSystemMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HostDatastoreSystemMO.java index f38f610e145..30798e31e19 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HostDatastoreSystemMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HostDatastoreSystemMO.java @@ -16,11 +16,7 @@ // under the License. package com.cloud.hypervisor.vmware.mo; -import java.net.URI; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - +import com.cloud.hypervisor.vmware.util.VmwareContext; import com.vmware.vim25.CustomFieldStringValue; import com.vmware.vim25.DatastoreInfo; import com.vmware.vim25.DynamicProperty; @@ -35,12 +31,18 @@ import com.vmware.vim25.ObjectContent; import com.vmware.vim25.ObjectSpec; import com.vmware.vim25.PropertyFilterSpec; import com.vmware.vim25.PropertySpec; +import com.vmware.vim25.RetrieveOptions; +import com.vmware.vim25.RetrieveResult; +import com.vmware.vim25.SelectionSpec; import com.vmware.vim25.TraversalSpec; import com.vmware.vim25.VmfsDatastoreCreateSpec; import com.vmware.vim25.VmfsDatastoreExpandSpec; import com.vmware.vim25.VmfsDatastoreOption; -import com.cloud.hypervisor.vmware.util.VmwareContext; +import java.net.URI; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; public class HostDatastoreSystemMO extends BaseMO { @@ -53,6 +55,14 @@ public class HostDatastoreSystemMO extends BaseMO { } public ManagedObjectReference findDatastore(String name) throws Exception { + ManagedObjectReference morDatastore = findSpecificDatastore(name); + if (morDatastore == null) { + morDatastore = findDatastoreCluster(name); + } + return morDatastore; + } + + public ManagedObjectReference findSpecificDatastore(String name) throws Exception { // added Apache CloudStack specific name convention, we will use custom field "cloud.uuid" as datastore name as well CustomFieldsManagerMO cfmMo = new CustomFieldsManagerMO(_context, _context.getServiceContent().getCustomFieldsManager()); int key = cfmMo.getCustomFieldKey("Datastore", CustomFieldConstants.CLOUD_UUID); @@ -79,6 +89,33 @@ public class HostDatastoreSystemMO extends BaseMO { return null; } + public ManagedObjectReference findDatastoreCluster(String name) throws Exception { + // added Apache CloudStack specific name convention, we will use custom field "cloud.uuid" as datastore name as well + CustomFieldsManagerMO cfmMo = new CustomFieldsManagerMO(_context, _context.getServiceContent().getCustomFieldsManager()); + int key = cfmMo.getCustomFieldKey("StoragePod", CustomFieldConstants.CLOUD_UUID); + assert (key != 0); + + List ocs = getDatastoreClusterPropertiesOnHostDatastoreSystem(new String[] {"name", String.format("value[%d]", key)}); + if (ocs != null) { + for (ObjectContent oc : ocs) { + if (oc.getPropSet().get(0).getVal().equals(name)) + return oc.getObj(); + + if (oc.getPropSet().size() > 1) { + DynamicProperty prop = oc.getPropSet().get(1); + if (prop != null && prop.getVal() != null) { + if (prop.getVal() instanceof CustomFieldStringValue) { + String val = ((CustomFieldStringValue)prop.getVal()).getValue(); + if (val.equalsIgnoreCase(name)) + return oc.getObj(); + } + } + } + } + } + return null; + } + public List queryUnresolvedVmfsVolumes() throws Exception { return _context.getService().queryUnresolvedVmfsVolumes(_mor); } @@ -251,4 +288,90 @@ public class HostDatastoreSystemMO extends BaseMO { return _context.getService().retrieveProperties(_context.getPropertyCollector(), pfSpecArr); } + + public List getDatastoreClusterPropertiesOnHostDatastoreSystem(String[] propertyPaths) throws Exception { + ManagedObjectReference retVal = null; + // Create Property Spec + PropertySpec propertySpec = new PropertySpec(); + propertySpec.setAll(Boolean.FALSE); + propertySpec.setType("StoragePod"); + propertySpec.getPathSet().addAll(Arrays.asList(propertyPaths)); + + // Now create Object Spec + ObjectSpec objectSpec = new ObjectSpec(); + objectSpec.setObj(getContext().getRootFolder()); + objectSpec.setSkip(Boolean.TRUE); + objectSpec.getSelectSet().addAll( + Arrays.asList(getStorageTraversalSpec())); + + // Create PropertyFilterSpec using the PropertySpec and ObjectPec + // created above. + PropertyFilterSpec propertyFilterSpec = new PropertyFilterSpec(); + propertyFilterSpec.getPropSet().add(propertySpec); + propertyFilterSpec.getObjectSet().add(objectSpec); + + List listpfs = new ArrayList(); + listpfs.add(propertyFilterSpec); + return retrievePropertiesAllObjects(listpfs); + } + + private SelectionSpec[] getStorageTraversalSpec() { + // create a traversal spec that start from root folder + + SelectionSpec ssFolders = new SelectionSpec(); + ssFolders.setName("visitFolders"); + + TraversalSpec datacenterSpec = new TraversalSpec(); + datacenterSpec.setName("dcTodf"); + datacenterSpec.setType("Datacenter"); + datacenterSpec.setPath("datastoreFolder"); + datacenterSpec.setSkip(Boolean.FALSE); + datacenterSpec.getSelectSet().add(ssFolders); + + TraversalSpec visitFolder = new TraversalSpec(); + visitFolder.setType("Folder"); + visitFolder.setName("visitFolders"); + visitFolder.setPath("childEntity"); + visitFolder.setSkip(Boolean.FALSE); + + List ssSpecList = new ArrayList(); + ssSpecList.add(datacenterSpec); + ssSpecList.add(ssFolders); + + visitFolder.getSelectSet().addAll(ssSpecList); + return (new SelectionSpec[]{visitFolder}); + } + + private List retrievePropertiesAllObjects( + List listpfs) throws Exception { + + RetrieveOptions propObjectRetrieveOpts = new RetrieveOptions(); + + List listobjcontent = new ArrayList(); + + RetrieveResult rslts = + getContext().getService().retrievePropertiesEx(getContext().getServiceContent().getPropertyCollector(), listpfs, + propObjectRetrieveOpts); + if (rslts != null && rslts.getObjects() != null + && !rslts.getObjects().isEmpty()) { + listobjcontent.addAll(rslts.getObjects()); + } + String token = null; + if (rslts != null && rslts.getToken() != null) { + token = rslts.getToken(); + } + while (token != null && !token.isEmpty()) { + rslts = + getContext().getService().continueRetrievePropertiesEx(getContext().getServiceContent().getPropertyCollector(), token); + token = null; + if (rslts != null) { + token = rslts.getToken(); + if (rslts.getObjects() != null && !rslts.getObjects().isEmpty()) { + listobjcontent.addAll(rslts.getObjects()); + } + } + } + return listobjcontent; + } + } diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HostMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HostMO.java index 8c02fb444f3..2483ced5cc5 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HostMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HostMO.java @@ -28,6 +28,7 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; +import com.cloud.storage.Storage; import com.google.gson.Gson; import com.vmware.vim25.AboutInfo; import com.vmware.vim25.AlreadyExistsFaultMsg; @@ -873,12 +874,15 @@ public class HostMO extends BaseMO implements VmwareHypervisorHost { } else { morDatastore = _context.getDatastoreMorByPath(poolPath); if (morDatastore == null) { - String msg = "Unable to create VMFS datastore. host: " + poolHostAddress + ", port: " + poolHostPort + ", path: " + poolPath + ", uuid: " + poolUuid; - s_logger.error(msg); + morDatastore = findDatastore(_context.getDatastoreNameFromPath(poolPath)); + if (morDatastore == null) { + String msg = "Unable to create VMFS datastore. host: " + poolHostAddress + ", port: " + poolHostPort + ", path: " + poolPath + ", uuid: " + poolUuid; + s_logger.error(msg); - if (s_logger.isTraceEnabled()) - s_logger.trace("vCenter API trace - mountDatastore() done(failed)"); - throw new Exception(msg); + if (s_logger.isTraceEnabled()) + s_logger.trace("vCenter API trace - mountDatastore() done(failed)"); + throw new Exception(msg); + } } dsMo = new DatastoreMO(_context, morDatastore); @@ -887,7 +891,7 @@ public class HostMO extends BaseMO implements VmwareHypervisorHost { } if (dsMo != null) { - HypervisorHostHelper.createBaseFolderInDatastore(dsMo, this); + HypervisorHostHelper.createBaseFolder(dsMo, this, "StoragePod".equals(morDatastore.getType()) ? Storage.StoragePoolType.DatastoreCluster: null); } if (s_logger.isTraceEnabled()) diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java index f25279bdee3..30ee117fe7b 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java @@ -54,6 +54,7 @@ import com.cloud.hypervisor.vmware.util.VmwareContext; import com.cloud.hypervisor.vmware.util.VmwareHelper; import com.cloud.network.Networks.BroadcastDomainType; import com.cloud.offering.NetworkOffering; +import com.cloud.storage.Storage.StoragePoolType; import com.cloud.utils.ActionDelegate; import com.cloud.utils.NumbersUtil; import com.cloud.utils.Pair; @@ -172,7 +173,6 @@ public class HypervisorHostHelper { if (morDs == null) morDs = hyperHost.findDatastore(uuidName); - DatastoreMO dsMo = new DatastoreMO(hyperHost.getContext(), morDs); return morDs; } @@ -2086,7 +2086,19 @@ public class HypervisorHostHelper { return DiskControllerType.getType(controller) == DiskControllerType.ide; } - public static void createBaseFolderInDatastore(DatastoreMO dsMo, VmwareHypervisorHost hyperHost) throws Exception { + public static void createBaseFolder(DatastoreMO dsMo, VmwareHypervisorHost hyperHost, StoragePoolType poolType) throws Exception { + if (poolType != null && poolType == StoragePoolType.DatastoreCluster) { + List datastoresInCluster = hyperHost.getContext().getVimClient().getDynamicProperty(dsMo.getMor(), "childEntity"); + for (ManagedObjectReference datastore : datastoresInCluster) { + DatastoreMO childDsMo = new DatastoreMO(hyperHost.getContext(), datastore); + createBaseFolderInDatastore(childDsMo, hyperHost); + } + } else { + createBaseFolderInDatastore(dsMo, hyperHost); + } + } + + private static void createBaseFolderInDatastore(DatastoreMO dsMo, VmwareHypervisorHost hyperHost) throws Exception { String dsPath = String.format("[%s]", dsMo.getName()); String folderPath = String.format("[%s] %s", dsMo.getName(), VSPHERE_DATASTORE_BASE_FOLDER); diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareContext.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareContext.java index 33b9644a307..807289f8bdf 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareContext.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareContext.java @@ -321,6 +321,24 @@ public class VmwareContext { return dcMo.findDatastore(tokens[1]); } + // path in format of / + public String getDatastoreNameFromPath(String inventoryPath) throws Exception { + assert (inventoryPath != null); + + String[] tokens; + if (inventoryPath.startsWith("/")) + tokens = inventoryPath.substring(1).split("/"); + else + tokens = inventoryPath.split("/"); + + if (tokens == null || tokens.length != 2) { + s_logger.error("Invalid datastore inventory path. path: " + inventoryPath); + return null; + } + + return tokens[1]; + } + public void waitForTaskProgressDone(ManagedObjectReference morTask) throws Exception { while (true) { TaskInfo tinfo = (TaskInfo)_vimClient.getDynamicProperty(morTask, "info"); From bea31b85eed21f20beff31469f0f5bcbad9041d8 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Tue, 14 Jul 2020 11:25:40 +0530 Subject: [PATCH 026/164] Assetion fix in StorageTest.java --- api/src/test/java/com/cloud/storage/StorageTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/test/java/com/cloud/storage/StorageTest.java b/api/src/test/java/com/cloud/storage/StorageTest.java index c57130f5418..61909e72e96 100644 --- a/api/src/test/java/com/cloud/storage/StorageTest.java +++ b/api/src/test/java/com/cloud/storage/StorageTest.java @@ -67,6 +67,6 @@ public class StorageTest { Assert.assertFalse(StoragePoolType.SMB.supportsOverProvisioning()); Assert.assertFalse(StoragePoolType.Gluster.supportsOverProvisioning()); Assert.assertFalse(StoragePoolType.ManagedNFS.supportsOverProvisioning()); - Assert.assertFalse(StoragePoolType.DatastoreCluster.supportsOverProvisioning()); + Assert.assertTrue(StoragePoolType.DatastoreCluster.supportsOverProvisioning()); } } From f858387b3c518af5a939416bfbdd833fbf76158f Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Wed, 15 Jul 2020 22:44:51 +0530 Subject: [PATCH 027/164] Import storagepolicies when vmware datacenter is added or updated --- .../vmware/manager/VmwareManagerImpl.java | 27 ++++++++++--------- .../vmware/manager/VmwareManagerImplTest.java | 1 + 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java index e10e88c19dd..a751abf1b8e 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java @@ -1184,6 +1184,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw } context = null; } + importVsphereStoragePoliciesInternal(zoneId, vmwareDc.getId()); return vmwareDc; } @@ -1244,6 +1245,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw hostDetailsDao.persist(host.getId(), hostDetails); } } + importVsphereStoragePoliciesInternal(zoneId, vmwareDc.getId()); return vmwareDc; } return null; @@ -1400,13 +1402,6 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw // Validate Id of zone doesZoneExist(zoneId); - // Get DC associated with this zone - VmwareDatacenterVO vmwareDatacenter; - String vmwareDcName; - String vCenterHost; - String userName; - String password; - DatacenterMO dcMo = null; final VmwareDatacenterZoneMapVO vmwareDcZoneMap = vmwareDatacenterZoneMapDao.findByZoneId(zoneId); // Check if zone is associated with VMware DC if (vmwareDcZoneMap == null) { @@ -1414,16 +1409,24 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw } final long vmwareDcId = vmwareDcZoneMap.getVmwareDcId(); - vmwareDatacenter = vmwareDcDao.findById(vmwareDcId); - vmwareDcName = vmwareDatacenter.getVmwareDatacenterName(); - vCenterHost = vmwareDatacenter.getVcenterHost(); - userName = vmwareDatacenter.getUser(); - password = vmwareDatacenter.getPassword(); + return importVsphereStoragePoliciesInternal(zoneId, vmwareDcId); + } + + public List importVsphereStoragePoliciesInternal(Long zoneId, Long vmwareDcId) { + + // Get DC associated with this zone + VmwareDatacenterVO vmwareDatacenter = vmwareDcDao.findById(vmwareDcId); + String vmwareDcName = vmwareDatacenter.getVmwareDatacenterName(); + String vCenterHost = vmwareDatacenter.getVcenterHost(); + String userName = vmwareDatacenter.getUser(); + String password = vmwareDatacenter.getPassword(); List storageProfiles = null; try { + s_logger.debug(String.format("Importing vSphere Storage Policies for the vmware DC %d in zone %d", vmwareDcId, zoneId)); VmwareContext context = VmwareContextFactory.getContext(vCenterHost, userName, password); PbmProfileManagerMO profileManagerMO = new PbmProfileManagerMO(context); storageProfiles = profileManagerMO.getStorageProfiles(); + s_logger.debug(String.format("Import vSphere Storage Policies for the vmware DC %d in zone %d is successful", vmwareDcId, zoneId)); } catch (Exception e) { String msg = String.format("Unable to list storage profiles from DC %s due to : %s", vmwareDcName, VmwareHelper.getExceptionMessage(e)); s_logger.error(msg); diff --git a/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImplTest.java b/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImplTest.java index 8aa92f7d9c4..80677e9d3ba 100644 --- a/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImplTest.java +++ b/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImplTest.java @@ -105,6 +105,7 @@ public class VmwareManagerImplTest { Mockito.lenient().doReturn(hostDetails).when(hostDetailsDao).findDetails(Mockito.anyLong()); Mockito.doReturn("some-old-guid").when(hostDetails).get("guid"); Mockito.doReturn(hostDetails).when(hostDetailsDao).findDetails(Mockito.anyLong()); + Mockito.doReturn(null).when(vmwareManager).importVsphereStoragePoliciesInternal(Mockito.anyLong(), Mockito.anyLong()); final VmwareDatacenter vmwareDatacenter = vmwareManager.updateVmwareDatacenter(updateVmwareDcCmd); From 41b3fc19d69eddd8a70b3d14d0668c805572c2ae Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Sun, 19 Jul 2020 15:36:39 +0530 Subject: [PATCH 028/164] Add Datastore cluster and the child entities which are datastores in the cluster into CloudStack Setting scope is still pending. --- .../com/cloud/agent/api/StoragePoolInfo.java | 21 +++++++ .../agent/api/ModifyStoragePoolAnswer.java | 11 +++- .../provider/DefaultHostListener.java | 61 +++++++++++++++++-- .../vmware/resource/VmwareResource.java | 45 ++++++++++++-- .../hypervisor/vmware/mo/DatastoreMO.java | 5 -- .../cloud/hypervisor/vmware/mo/HostMO.java | 5 +- .../vmware/mo/HypervisorHostHelper.java | 5 +- .../hypervisor/vmware/mo/StoragepodMO.java | 48 +++++++++++++++ 8 files changed, 179 insertions(+), 22 deletions(-) create mode 100644 vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/StoragepodMO.java diff --git a/api/src/main/java/com/cloud/agent/api/StoragePoolInfo.java b/api/src/main/java/com/cloud/agent/api/StoragePoolInfo.java index 7d1e2389070..d923694a854 100644 --- a/api/src/main/java/com/cloud/agent/api/StoragePoolInfo.java +++ b/api/src/main/java/com/cloud/agent/api/StoragePoolInfo.java @@ -28,6 +28,7 @@ public class StoragePoolInfo { StoragePoolType poolType; long capacityBytes; long availableBytes; + String name; Map details; protected StoragePoolInfo() { @@ -67,14 +68,34 @@ public class StoragePoolInfo { return host; } + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public void setLocalPath(String localPath) { + this.localPath = localPath; + } + public String getLocalPath() { return localPath; } + public void setHostPath(String hostPath) { + this.hostPath = hostPath; + } + public String getHostPath() { return hostPath; } + public void setPoolType(StoragePoolType poolType) { + this.poolType = poolType; + } + public StoragePoolType getPoolType() { return poolType; } diff --git a/core/src/main/java/com/cloud/agent/api/ModifyStoragePoolAnswer.java b/core/src/main/java/com/cloud/agent/api/ModifyStoragePoolAnswer.java index a5f7c53214b..f7a7e1ced06 100644 --- a/core/src/main/java/com/cloud/agent/api/ModifyStoragePoolAnswer.java +++ b/core/src/main/java/com/cloud/agent/api/ModifyStoragePoolAnswer.java @@ -19,6 +19,7 @@ package com.cloud.agent.api; +import java.util.List; import java.util.Map; import com.cloud.storage.template.TemplateProp; @@ -28,6 +29,7 @@ public class ModifyStoragePoolAnswer extends Answer { private Map templateInfo; private String localDatastoreName; private String poolType; + private List datastoreClusterChildren; public ModifyStoragePoolAnswer(ModifyStoragePoolCommand cmd, long capacityBytes, long availableBytes, Map tInfo) { super(cmd); @@ -63,7 +65,6 @@ public class ModifyStoragePoolAnswer extends Answer { return localDatastoreName; } - public String getPoolType() { return poolType; } @@ -71,4 +72,12 @@ public class ModifyStoragePoolAnswer extends Answer { public void setPoolType(String poolType) { this.poolType = poolType; } + + public List getDatastoreClusterChildren() { + return datastoreClusterChildren; + } + + public void setDatastoreClusterChildren(List datastoreClusterChildren) { + this.datastoreClusterChildren = datastoreClusterChildren; + } } diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java index 3ba45191147..dac6514cc84 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java @@ -22,12 +22,15 @@ import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; import com.cloud.agent.api.ModifyStoragePoolAnswer; import com.cloud.agent.api.ModifyStoragePoolCommand; +import com.cloud.agent.api.StoragePoolInfo; import com.cloud.alert.AlertManager; import com.cloud.exception.StorageConflictException; import com.cloud.storage.DataStoreRole; import com.cloud.storage.StoragePool; import com.cloud.storage.StoragePoolHostVO; +import com.cloud.storage.StoragePoolStatus; import com.cloud.storage.dao.StoragePoolHostDao; +import com.cloud.storage.dao.StoragePoolTagsDao; import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.HypervisorHostListener; @@ -39,7 +42,9 @@ import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import javax.inject.Inject; +import java.util.HashMap; import java.util.List; +import java.util.Map; public class DefaultHostListener implements HypervisorHostListener { private static final Logger s_logger = Logger.getLogger(DefaultHostListener.class); @@ -55,6 +60,8 @@ public class DefaultHostListener implements HypervisorHostListener { PrimaryDataStoreDao primaryStoreDao; @Inject StoragePoolDetailsDao storagePoolDetailsDao; + @Inject + StoragePoolTagsDao storagePoolTagsDao; @Override public boolean hostAdded(long hostId) { @@ -92,7 +99,52 @@ public class DefaultHostListener implements HypervisorHostListener { } } } + StoragePoolVO poolVO = this.primaryStoreDao.findById(poolId); + updateStoragePoolHostVOAndDetails(poolVO, hostId, mspAnswer); + for (ModifyStoragePoolAnswer childDataStoreAnswer : ((ModifyStoragePoolAnswer) answer).getDatastoreClusterChildren()) { + StoragePoolInfo childStoragePoolInfo = childDataStoreAnswer.getPoolInfo(); + StoragePoolVO dataStoreVO = primaryStoreDao.findPoolByUUID(childStoragePoolInfo.getUuid()); + if (dataStoreVO != null) { + continue; + } + dataStoreVO = new StoragePoolVO(); + dataStoreVO.setStorageProviderName(poolVO.getStorageProviderName()); + dataStoreVO.setHostAddress(childStoragePoolInfo.getHost()); + dataStoreVO.setPoolType(poolVO.getPoolType()); + dataStoreVO.setPath(childStoragePoolInfo.getHostPath()); + dataStoreVO.setPort(poolVO.getPort()); + dataStoreVO.setName(childStoragePoolInfo.getName()); + dataStoreVO.setUuid(childStoragePoolInfo.getUuid()); + dataStoreVO.setDataCenterId(poolVO.getDataCenterId()); + dataStoreVO.setPodId(poolVO.getPodId()); + dataStoreVO.setClusterId(poolVO.getClusterId()); + dataStoreVO.setStatus(StoragePoolStatus.Up); + dataStoreVO.setUserInfo(poolVO.getUserInfo()); + dataStoreVO.setManaged(poolVO.isManaged()); + dataStoreVO.setCapacityIops(poolVO.getCapacityIops()); + dataStoreVO.setCapacityBytes(childDataStoreAnswer.getPoolInfo().getCapacityBytes()); + dataStoreVO.setUsedBytes(childDataStoreAnswer.getPoolInfo().getCapacityBytes() - childDataStoreAnswer.getPoolInfo().getAvailableBytes()); + dataStoreVO.setHypervisor(poolVO.getHypervisor()); + dataStoreVO.setScope(poolVO.getScope()); + + Map details = new HashMap<>(); + if(StringUtils.isNotEmpty(childDataStoreAnswer.getPoolType())) { + details.put("pool_type", childDataStoreAnswer.getPoolType()); + } + + List storageTags = storagePoolTagsDao.getStoragePoolTags(poolId); + storageTags.add("DataStoreCluster-" + poolVO.getUuid()); + primaryStoreDao.persist(dataStoreVO, details, storageTags); + + updateStoragePoolHostVOAndDetails(dataStoreVO, hostId, childDataStoreAnswer); + } + + s_logger.info("Connection established between storage pool " + pool + " and host " + hostId); + return true; + } + + private void updateStoragePoolHostVOAndDetails(StoragePool pool, long hostId, ModifyStoragePoolAnswer mspAnswer) { StoragePoolHostVO poolHost = storagePoolHostDao.findByPoolHost(pool.getId(), hostId); if (poolHost == null) { poolHost = new StoragePoolHostVO(pool.getId(), hostId, mspAnswer.getPoolInfo().getLocalPath().replaceAll("//", "/")); @@ -101,20 +153,17 @@ public class DefaultHostListener implements HypervisorHostListener { poolHost.setLocalPath(mspAnswer.getPoolInfo().getLocalPath().replaceAll("//", "/")); } - StoragePoolVO poolVO = this.primaryStoreDao.findById(poolId); + StoragePoolVO poolVO = this.primaryStoreDao.findById(pool.getId()); poolVO.setUsedBytes(mspAnswer.getPoolInfo().getCapacityBytes() - mspAnswer.getPoolInfo().getAvailableBytes()); poolVO.setCapacityBytes(mspAnswer.getPoolInfo().getCapacityBytes()); if(StringUtils.isNotEmpty(mspAnswer.getPoolType())) { - StoragePoolDetailVO poolType = storagePoolDetailsDao.findDetail(poolId, "pool_type"); + StoragePoolDetailVO poolType = storagePoolDetailsDao.findDetail(pool.getId(), "pool_type"); if (poolType == null) { - StoragePoolDetailVO storagePoolDetailVO = new StoragePoolDetailVO(poolId, "pool_type", mspAnswer.getPoolType(), false); + StoragePoolDetailVO storagePoolDetailVO = new StoragePoolDetailVO(pool.getId(), "pool_type", mspAnswer.getPoolType(), false); storagePoolDetailsDao.persist(storagePoolDetailVO); } } primaryStoreDao.update(pool.getId(), poolVO); - - s_logger.info("Connection established between storage pool " + pool + " and host " + hostId); - return true; } @Override diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index ffeebc7c5f3..0c5fc48ded1 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -177,6 +177,7 @@ import com.cloud.hypervisor.vmware.mo.HostStorageSystemMO; import com.cloud.hypervisor.vmware.mo.HypervisorHostHelper; import com.cloud.hypervisor.vmware.mo.NetworkDetails; import com.cloud.hypervisor.vmware.mo.TaskMO; +import com.cloud.hypervisor.vmware.mo.StoragepodMO; import com.cloud.hypervisor.vmware.mo.VirtualEthernetCardType; import com.cloud.hypervisor.vmware.mo.VirtualMachineDiskInfoBuilder; import com.cloud.hypervisor.vmware.mo.VirtualMachineMO; @@ -4918,11 +4919,43 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa long capacity = 0; long available = 0; + List childDatastoresModifyStoragePoolAnswers = new ArrayList<>(); if (pool.getType() == StoragePoolType.DatastoreCluster) { - StoragePodSummary summary = dsMo.getDatastoreClusterSummary(); - capacity = summary.getCapacity(); - available = summary.getFreeSpace(); + StoragepodMO datastoreClusterMo = new StoragepodMO(getServiceContext(), morDatastore); + StoragePodSummary dsClusterSummary = datastoreClusterMo.getDatastoreClusterSummary(); + capacity = dsClusterSummary.getCapacity(); + available = dsClusterSummary.getFreeSpace(); + + List childDatastoreMors = datastoreClusterMo.getDatastoresInDatastoreCluster(); + for (ManagedObjectReference childDsMor : childDatastoreMors) { + DatastoreMO childDsMo = new DatastoreMO(getServiceContext(), childDsMor); + + Map tInfo = new HashMap<>(); + DatastoreSummary summary = childDsMo.getDatastoreSummary();; + ModifyStoragePoolAnswer answer = new ModifyStoragePoolAnswer(cmd, summary.getCapacity(), summary.getFreeSpace(), tInfo); + StoragePoolInfo poolInfo = answer.getPoolInfo(); + poolInfo.setName(summary.getName()); + String datastoreClusterPath = pool.getPath(); + int pathstartPosition = datastoreClusterPath.lastIndexOf('/'); + String datacenterName = datastoreClusterPath.substring(0, pathstartPosition+1); + String childPath = datacenterName + summary.getName(); + poolInfo.setHostPath(childPath); + String uuid = UUID.nameUUIDFromBytes(((pool.getHost() + childPath)).getBytes()).toString(); + poolInfo.setUuid(uuid); + poolInfo.setLocalPath(cmd.LOCAL_PATH_PREFIX + File.separator + uuid); + + answer.setPoolInfo(poolInfo); + answer.setPoolType(summary.getType()); + answer.setLocalDatastoreName(morDatastore.getValue()); + + childDsMo.setCustomFieldValue(CustomFieldConstants.CLOUD_UUID, uuid); + HypervisorHostHelper.createBaseFolderInDatastore(childDsMo, hyperHost); + + childDatastoresModifyStoragePoolAnswers.add(answer); + } } else { + HypervisorHostHelper.createBaseFolderInDatastore(dsMo, hyperHost); + DatastoreSummary summary = dsMo.getDatastoreSummary(); capacity = summary.getCapacity(); available = summary.getFreeSpace(); @@ -4930,6 +4963,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa Map tInfo = new HashMap<>(); ModifyStoragePoolAnswer answer = new ModifyStoragePoolAnswer(cmd, capacity, available, tInfo); + answer.setDatastoreClusterChildren(childDatastoresModifyStoragePoolAnswers); if (cmd.getAdd() && (pool.getType() == StoragePoolType.VMFS || pool.getType() == StoragePoolType.PreSetup) && pool.getType() != StoragePoolType.DatastoreCluster) { answer.setPoolType(dsMo.getDatastoreType()); @@ -5312,14 +5346,15 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa ManagedObjectReference morDs = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, cmd.getStorageId()); if (morDs != null) { - DatastoreMO datastoreMo = new DatastoreMO(context, morDs); long capacity = 0; long free = 0; if (cmd.getPooltype() == StoragePoolType.DatastoreCluster) { - StoragePodSummary summary = datastoreMo.getDatastoreClusterSummary(); + StoragepodMO datastoreClusterMo = new StoragepodMO(getServiceContext(), morDs); + StoragePodSummary summary = datastoreClusterMo.getDatastoreClusterSummary(); capacity = summary.getCapacity(); free = summary.getFreeSpace(); } else { + DatastoreMO datastoreMo = new DatastoreMO(context, morDs); DatastoreSummary summary = datastoreMo.getDatastoreSummary(); capacity = summary.getCapacity(); free = summary.getFreeSpace(); diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatastoreMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatastoreMO.java index 133a412d750..b5955a6d709 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatastoreMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatastoreMO.java @@ -33,7 +33,6 @@ import com.vmware.vim25.ObjectSpec; import com.vmware.vim25.PropertyFilterSpec; import com.vmware.vim25.PropertySpec; import com.vmware.vim25.SelectionSpec; -import com.vmware.vim25.StoragePodSummary; import com.vmware.vim25.TraversalSpec; import org.apache.log4j.Logger; @@ -68,10 +67,6 @@ public class DatastoreMO extends BaseMO { return (DatastoreSummary)_context.getVimClient().getDynamicProperty(_mor, "summary"); } - public StoragePodSummary getDatastoreClusterSummary() throws Exception { - return (StoragePodSummary)_context.getVimClient().getDynamicProperty(_mor, "summary"); - } - public HostDatastoreBrowserMO getHostDatastoreBrowserMO() throws Exception { return new HostDatastoreBrowserMO(_context, (ManagedObjectReference)_context.getVimClient().getDynamicProperty(_mor, "browser")); } diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HostMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HostMO.java index 2483ced5cc5..6554440a3e5 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HostMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HostMO.java @@ -28,7 +28,6 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; -import com.cloud.storage.Storage; import com.google.gson.Gson; import com.vmware.vim25.AboutInfo; import com.vmware.vim25.AlreadyExistsFaultMsg; @@ -890,8 +889,8 @@ public class HostMO extends BaseMO implements VmwareHypervisorHost { } } - if (dsMo != null) { - HypervisorHostHelper.createBaseFolder(dsMo, this, "StoragePod".equals(morDatastore.getType()) ? Storage.StoragePoolType.DatastoreCluster: null); + if (dsMo != null && !"StoragePod".equals(morDatastore.getType())) { + HypervisorHostHelper.createBaseFolderInDatastore(dsMo, this); } if (s_logger.isTraceEnabled()) diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java index 30ee117fe7b..7dcc5c77f5d 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java @@ -2088,7 +2088,8 @@ public class HypervisorHostHelper { public static void createBaseFolder(DatastoreMO dsMo, VmwareHypervisorHost hyperHost, StoragePoolType poolType) throws Exception { if (poolType != null && poolType == StoragePoolType.DatastoreCluster) { - List datastoresInCluster = hyperHost.getContext().getVimClient().getDynamicProperty(dsMo.getMor(), "childEntity"); + StoragepodMO storagepodMO = new StoragepodMO(hyperHost.getContext(), dsMo.getMor()); + List datastoresInCluster = storagepodMO.getDatastoresInDatastoreCluster(); for (ManagedObjectReference datastore : datastoresInCluster) { DatastoreMO childDsMo = new DatastoreMO(hyperHost.getContext(), datastore); createBaseFolderInDatastore(childDsMo, hyperHost); @@ -2098,7 +2099,7 @@ public class HypervisorHostHelper { } } - private static void createBaseFolderInDatastore(DatastoreMO dsMo, VmwareHypervisorHost hyperHost) throws Exception { + public static void createBaseFolderInDatastore(DatastoreMO dsMo, VmwareHypervisorHost hyperHost) throws Exception { String dsPath = String.format("[%s]", dsMo.getName()); String folderPath = String.format("[%s] %s", dsMo.getName(), VSPHERE_DATASTORE_BASE_FOLDER); diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/StoragepodMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/StoragepodMO.java new file mode 100644 index 00000000000..afa3a022126 --- /dev/null +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/StoragepodMO.java @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.hypervisor.vmware.mo; + +import com.cloud.hypervisor.vmware.util.VmwareContext; +import com.vmware.vim25.ManagedObjectReference; +import com.vmware.vim25.StoragePodSummary; +import org.apache.log4j.Logger; + +import java.util.List; + +public class StoragepodMO extends BaseMO { + + private static final Logger LOGGER = Logger.getLogger(StoragepodMO.class); + + public StoragepodMO(VmwareContext context, ManagedObjectReference mor) { + super(context, mor); + } + + public StoragepodMO(VmwareContext context, String morType, String morValue) { + super(context, morType, morValue); + } + + public StoragePodSummary getDatastoreClusterSummary() throws Exception { + return (StoragePodSummary)_context.getVimClient().getDynamicProperty(_mor, "summary"); + } + + public List getDatastoresInDatastoreCluster() throws Exception { + List datastoresInCluster = _context.getVimClient().getDynamicProperty(_mor, "childEntity"); + return datastoresInCluster; + } + +} From 53fa3beecf042c0ad6c7444e007d327f0c454fff Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Tue, 21 Jul 2020 12:37:59 +0530 Subject: [PATCH 029/164] Register Disk to first class disk whereever possible --- .../vmware/resource/VmwareResource.java | 23 ++++++++ .../vmware/mo/VirtualMachineMO.java | 54 +++++++++++++++---- 2 files changed, 68 insertions(+), 9 deletions(-) diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 0c5fc48ded1..f11342cd846 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -181,6 +181,7 @@ import com.cloud.hypervisor.vmware.mo.StoragepodMO; import com.cloud.hypervisor.vmware.mo.VirtualEthernetCardType; import com.cloud.hypervisor.vmware.mo.VirtualMachineDiskInfoBuilder; import com.cloud.hypervisor.vmware.mo.VirtualMachineMO; +import com.cloud.hypervisor.vmware.mo.VirtualStorageObjectManagerMO; import com.cloud.hypervisor.vmware.mo.VirtualSwitchType; import com.cloud.hypervisor.vmware.mo.VmwareHypervisorHost; import com.cloud.hypervisor.vmware.mo.VmwareHypervisorHostNetworkSummary; @@ -226,6 +227,7 @@ import com.cloud.vm.VmDetailConstants; import com.google.common.base.Strings; import com.google.gson.Gson; import com.vmware.vim25.AboutInfo; +import com.vmware.vim25.AlreadyExistsFaultMsg; import com.vmware.vim25.ArrayUpdateOperation; import com.vmware.vim25.BoolPolicy; import com.vmware.vim25.ComputeResourceSummary; @@ -245,6 +247,7 @@ import com.vmware.vim25.HostCapability; import com.vmware.vim25.HostHostBusAdapter; import com.vmware.vim25.HostInternetScsiHba; import com.vmware.vim25.HostPortGroupSpec; +import com.vmware.vim25.ID; import com.vmware.vim25.ManagedObjectReference; import com.vmware.vim25.NasDatastoreInfo; import com.vmware.vim25.ObjectContent; @@ -266,6 +269,8 @@ import com.vmware.vim25.VAppProductSpec; import com.vmware.vim25.VAppPropertyInfo; import com.vmware.vim25.VAppPropertySpec; import com.vmware.vim25.VMwareDVSPortSetting; +import com.vmware.vim25.VStorageObject; +import com.vmware.vim25.VStorageObjectConfigInfo; import com.vmware.vim25.VimPortType; import com.vmware.vim25.VirtualDevice; import com.vmware.vim25.VirtualDeviceBackingInfo; @@ -746,6 +751,24 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } else if (newSize == oldSize) { return new ResizeVolumeAnswer(cmd, true, "success", newSize * ResourceType.bytesToKiB); } + /* + // FR41 this is yet to fix + ManagedObjectReference morDS1 = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, cmd.getPoolUuid()); + DatastoreMO dsMo1 = new DatastoreMO(hyperHost.getContext(), morDS1); + vmdkDataStorePath = VmwareStorageLayoutHelper.getLegacyDatastorePathFromVmdkFileName(dsMo1, path + VMDK_EXTENSION); + DatastoreFile dsFile1 = new DatastoreFile(vmdkDataStorePath); + + s_logger.debug("vDiskid does not exist for volume " + vmdkDataStorePath + " registering the disk now"); + VirtualStorageObjectManagerMO vStorageObjectManagerMO = new VirtualStorageObjectManagerMO(getServiceContext()); + try { + VStorageObject vStorageObject = vStorageObjectManagerMO.registerVirtualDisk(dsFile1, null, dsMo1.getOwnerDatacenter().second()); + VStorageObjectConfigInfo diskConfigInfo = vStorageObject.getConfig(); + ID vdiskId = diskConfigInfo.getId(); + } catch (Throwable e) { + if (e instanceof AlreadyExistsFaultMsg) { + + } + }*/ if (vmName.equalsIgnoreCase("none")) { // OfflineVmwareMigration: we need to refactor the worker vm creation out for use in migration methods as well as here diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java index 3100b7f26ff..0382f21df60 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java @@ -2436,13 +2436,7 @@ public class VirtualMachineMO extends BaseMO { String deviceNumbering = getDeviceBusName(devices, device); s_logger.info("Disk backing : " + diskBackingInfo.getFileName() + " matches ==> " + deviceNumbering); - if (((VirtualDisk) device).getVDiskId() == null) { - s_logger.debug("vDiskid does not exist for volume " + vmdkDatastorePath + " registering the disk now"); - VirtualStorageObjectManagerMO vStorageObjectManagerMO = new VirtualStorageObjectManagerMO(getOwnerDatacenter().first().getContext()); - VStorageObject vStorageObject = vStorageObjectManagerMO.registerVirtualDisk(dsBackingFile, null, getOwnerDatacenter().first().getName()); - VStorageObjectConfigInfo diskConfigInfo = vStorageObject.getConfig(); - ((VirtualDisk) device).setVDiskId(diskConfigInfo.getId()); - } + registerVirtualDisk((VirtualDisk) device, dsBackingFile); return new Pair<>((VirtualDisk)device, deviceNumbering); } @@ -2513,15 +2507,17 @@ public class VirtualMachineMO extends BaseMO { if (matchExactly) { if (backingBaseName.equalsIgnoreCase(srcBaseName)) { String deviceNumbering = getDeviceBusName(devices, device); - s_logger.info("Disk backing : " + diskBackingInfo.getFileName() + " matches ==> " + deviceNumbering); + + registerVirtualDisk((VirtualDisk) device, dsBackingFile); return new Pair((VirtualDisk)device, deviceNumbering); } } else { if (backingBaseName.contains(trimmedSrcBaseName)) { String deviceNumbering = getDeviceBusName(devices, device); - s_logger.info("Disk backing : " + diskBackingInfo.getFileName() + " matches ==> " + deviceNumbering); + + registerVirtualDisk((VirtualDisk) device, dsBackingFile); return new Pair((VirtualDisk)device, deviceNumbering); } } @@ -2536,6 +2532,20 @@ public class VirtualMachineMO extends BaseMO { return null; } + public void registerVirtualDisk(VirtualDisk device, DatastoreFile dsBackingFile) { + if (((VirtualDisk) device).getVDiskId() == null) { + try { + s_logger.debug("vDiskid does not exist for volume " + dsBackingFile.getFileName() + " registering the disk now"); + VirtualStorageObjectManagerMO vStorageObjectManagerMO = new VirtualStorageObjectManagerMO(getOwnerDatacenter().first().getContext()); + VStorageObject vStorageObject = vStorageObjectManagerMO.registerVirtualDisk(dsBackingFile, null, getOwnerDatacenter().first().getName()); + VStorageObjectConfigInfo diskConfigInfo = vStorageObject.getConfig(); + ((VirtualDisk) device).setVDiskId(diskConfigInfo.getId()); + } catch (Exception e) { + s_logger.warn("Exception while trying to register a disk as first class disk to get the unique identifier, main operation still continues: " + e.getMessage()); + } + } + } + public String getDiskCurrentTopBackingFileInChain(String deviceBusName) throws Exception { List devices = _context.getVimClient().getDynamicProperty(_mor, "config.hardware.device"); if (devices != null && devices.size() > 0) { @@ -2591,6 +2601,8 @@ public class VirtualMachineMO extends BaseMO { builder.addDisk(deviceBusName, diskBackingInfo.getFileName()); diskBackingInfo = diskBackingInfo.getParent(); } + DatastoreFile dsBackingFile = new DatastoreFile(diskBackingInfo.getFileName()); + registerVirtualDisk((VirtualDisk) device, dsBackingFile); } } } @@ -2609,6 +2621,8 @@ public class VirtualMachineMO extends BaseMO { VirtualDeviceBackingInfo backingInfo = ((VirtualDisk)device).getBacking(); if (backingInfo instanceof VirtualDiskFlatVer2BackingInfo) { VirtualDiskFlatVer2BackingInfo diskBackingInfo = (VirtualDiskFlatVer2BackingInfo)backingInfo; + DatastoreFile dsBackingFile = new DatastoreFile(diskBackingInfo.getFileName()); + registerVirtualDisk((VirtualDisk) device, dsBackingFile); disks.add(new Pair(new Integer(device.getKey()), diskBackingInfo.getDatastore())); } } @@ -2717,6 +2731,10 @@ public class VirtualMachineMO extends BaseMO { for (VirtualDevice device : devices) { if (device instanceof VirtualDisk) { + VirtualDeviceBackingInfo backingInfo = device.getBacking(); + VirtualDiskFlatVer2BackingInfo diskBackingInfo = (VirtualDiskFlatVer2BackingInfo)backingInfo; + DatastoreFile dsBackingFile = new DatastoreFile(diskBackingInfo.getFileName()); + registerVirtualDisk((VirtualDisk) device, dsBackingFile); virtualDisks.add((VirtualDisk)device); } } @@ -2751,6 +2769,7 @@ public class VirtualMachineMO extends BaseMO { reConfigSpec.getDeviceChange().add(deviceConfigSpec); } + registerVirtualDisk((VirtualDisk) device, dsBackingFile); } } @@ -2778,6 +2797,23 @@ public class VirtualMachineMO extends BaseMO { if (devices != null && devices.size() > 0) { for (VirtualDevice device : devices) { if (device instanceof VirtualDisk) { + if (((VirtualDisk) device).getVDiskId() == null) { + try { + // Register as first class disk + VirtualDeviceBackingInfo backingInfo = device.getBacking(); + if (backingInfo instanceof VirtualDiskFlatVer2BackingInfo) { + VirtualDiskFlatVer2BackingInfo diskBackingInfo = (VirtualDiskFlatVer2BackingInfo) backingInfo; + DatastoreFile dsBackingFile = new DatastoreFile(diskBackingInfo.getFileName()); + s_logger.debug("vDiskid does not exist for volume " + diskBackingInfo.getFileName() + " registering the disk now"); + VirtualStorageObjectManagerMO vStorageObjectManagerMO = new VirtualStorageObjectManagerMO(getOwnerDatacenter().first().getContext()); + VStorageObject vStorageObject = vStorageObjectManagerMO.registerVirtualDisk(dsBackingFile, null, getOwnerDatacenter().first().getName()); + VStorageObjectConfigInfo diskConfigInfo = vStorageObject.getConfig(); + ((VirtualDisk) device).setVDiskId(diskConfigInfo.getId()); + } + } catch (Exception e) { + s_logger.warn("Exception while trying to register a disk as first class disk to get the unique identifier, main operation still continues: " + e.getMessage()); + } + } deviceList.add((VirtualDisk)device); } } From b4a23ea5f6255af7e217c890bf6edbc7cb63cf0c Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Tue, 21 Jul 2020 22:08:00 +0530 Subject: [PATCH 030/164] Allocation logic to skip datastore cluster and consider only storagepools inside the datastore cluster --- .../main/java/com/cloud/storage/StorageManager.java | 2 ++ .../storage/datastore/db/PrimaryDataStoreDao.java | 2 ++ .../storage/datastore/db/PrimaryDataStoreDaoImpl.java | 6 ++++++ .../storage/datastore/db/StoragePoolVO.java | 11 +++++++++++ .../allocator/AbstractStoragePoolAllocator.java | 5 +++++ .../java/com/cloud/storage/StorageManagerImpl.java | 9 +++++++++ 6 files changed, 35 insertions(+) diff --git a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java index b63574cd808..0e245920af3 100644 --- a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java +++ b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java @@ -238,4 +238,6 @@ public interface StorageManager extends StorageService { DiskTO getDiskWithThrottling(DataTO volTO, Volume.Type volumeType, long deviceId, String path, long offeringId, long diskOfferingId); + boolean isStoragePoolDatastoreClusterParent(StoragePool pool); + } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java index 2398e91c90c..f5929f440c4 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java @@ -123,4 +123,6 @@ public interface PrimaryDataStoreDao extends GenericDao { List listLocalStoragePoolByPath(long datacenterId, String path); void deletePoolTags(long poolId); + + List listChildStoragePoolsInDatastoreCluster(long poolId); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java index fee9dc28bd0..7d6b2821e72 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java @@ -552,4 +552,10 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase public void deletePoolTags(long poolId) { _tagsDao.deleteTags(poolId); } + + @Override + public List listChildStoragePoolsInDatastoreCluster(long poolId) { + QueryBuilder sc = QueryBuilder.create(StoragePoolVO.class); + sc.and(sc.entity().getParent(), Op.EQ, poolId); + return sc.list(); } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/StoragePoolVO.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/StoragePoolVO.java index 24fcaa03f56..2cdcae54de2 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/StoragePoolVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/StoragePoolVO.java @@ -119,6 +119,9 @@ public class StoragePoolVO implements StoragePool { @Enumerated(value = EnumType.STRING) private HypervisorType hypervisor; + @Column(name = "parent") + private Long parent; + @Override public long getId() { return id; @@ -373,6 +376,14 @@ public class StoragePoolVO implements StoragePool { return !isShared(); } + public Long getParent() { + return parent; + } + + public void setParent(Long parent) { + this.parent = parent; + } + @Override public boolean isInMaintenance() { return status == StoragePoolStatus.PrepareForMaintenance || status == StoragePoolStatus.Maintenance || status == StoragePoolStatus.ErrorInMaintenance || diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java index 23fb2142868..68efe16fd7a 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java @@ -217,6 +217,11 @@ public abstract class AbstractStoragePoolAllocator extends AdapterBase implement List requestVolumes = new ArrayList<>(); requestVolumes.add(volume); if (dskCh.getHypervisorType() == HypervisorType.VMware) { + // Skip the parent datastore cluster, consider only child storage pools in it + if (pool.getPoolType() == Storage.StoragePoolType.DatastoreCluster && storageMgr.isStoragePoolDatastoreClusterParent(pool)) { + return false; + } + try { boolean isStoragePoolStoragepolicyComplaince = storageMgr.isStoragePoolComplaintWithStoragePolicy(requestVolumes, pool); if (!isStoragePoolStoragepolicyComplaince) { diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index 4f28fa29e72..b8a2f83d822 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -2613,6 +2613,15 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C return disk; } + @Override + public boolean isStoragePoolDatastoreClusterParent(StoragePool pool) { + List childStoragePools = _storagePoolDao.listChildStoragePoolsInDatastoreCluster(pool.getId()); + if (childStoragePools != null && !childStoragePools.isEmpty()) { + return true; + } + return false; + } + private void setVolumeObjectTOThrottling(VolumeObjectTO volumeTO, final ServiceOffering offering, final DiskOffering diskOffering) { volumeTO.setBytesReadRate(getDiskBytesReadRate(offering, diskOffering)); volumeTO.setBytesWriteRate(getDiskBytesWriteRate(offering, diskOffering)); From c9c38c0f37be784efd341374ceed05d167dc9995 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Wed, 22 Jul 2020 00:18:51 +0530 Subject: [PATCH 031/164] Handle datastore clusters in list storage pools to list only parent not the children in the Datastore cluster --- .../storage/datastore/db/PrimaryDataStoreDao.java | 2 ++ .../storage/datastore/db/PrimaryDataStoreDaoImpl.java | 11 ++++++++++- .../hypervisor/vmware/resource/VmwareResource.java | 5 ----- .../java/com/cloud/api/query/QueryManagerImpl.java | 2 ++ .../com/cloud/api/query/vo/StoragePoolJoinVO.java | 7 +++++++ 5 files changed, 21 insertions(+), 6 deletions(-) diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java index f5929f440c4..5712411a48e 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java @@ -125,4 +125,6 @@ public interface PrimaryDataStoreDao extends GenericDao { void deletePoolTags(long poolId); List listChildStoragePoolsInDatastoreCluster(long poolId); + + Integer countAll(); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java index 7d6b2821e72..1d1e0a0198d 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java @@ -557,5 +557,14 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase public List listChildStoragePoolsInDatastoreCluster(long poolId) { QueryBuilder sc = QueryBuilder.create(StoragePoolVO.class); sc.and(sc.entity().getParent(), Op.EQ, poolId); - return sc.list(); } + return sc.list(); + } + + @Override + public Integer countAll() { + SearchCriteria sc = createSearchCriteria(); + sc.addAnd("parent", SearchCriteria.Op.EQ, 0); + sc.addAnd("removed", SearchCriteria.Op.NULL); + return getCount(sc); + } } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index f11342cd846..577e1a6e0af 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -181,7 +181,6 @@ import com.cloud.hypervisor.vmware.mo.StoragepodMO; import com.cloud.hypervisor.vmware.mo.VirtualEthernetCardType; import com.cloud.hypervisor.vmware.mo.VirtualMachineDiskInfoBuilder; import com.cloud.hypervisor.vmware.mo.VirtualMachineMO; -import com.cloud.hypervisor.vmware.mo.VirtualStorageObjectManagerMO; import com.cloud.hypervisor.vmware.mo.VirtualSwitchType; import com.cloud.hypervisor.vmware.mo.VmwareHypervisorHost; import com.cloud.hypervisor.vmware.mo.VmwareHypervisorHostNetworkSummary; @@ -227,7 +226,6 @@ import com.cloud.vm.VmDetailConstants; import com.google.common.base.Strings; import com.google.gson.Gson; import com.vmware.vim25.AboutInfo; -import com.vmware.vim25.AlreadyExistsFaultMsg; import com.vmware.vim25.ArrayUpdateOperation; import com.vmware.vim25.BoolPolicy; import com.vmware.vim25.ComputeResourceSummary; @@ -247,7 +245,6 @@ import com.vmware.vim25.HostCapability; import com.vmware.vim25.HostHostBusAdapter; import com.vmware.vim25.HostInternetScsiHba; import com.vmware.vim25.HostPortGroupSpec; -import com.vmware.vim25.ID; import com.vmware.vim25.ManagedObjectReference; import com.vmware.vim25.NasDatastoreInfo; import com.vmware.vim25.ObjectContent; @@ -269,8 +266,6 @@ import com.vmware.vim25.VAppProductSpec; import com.vmware.vim25.VAppPropertyInfo; import com.vmware.vim25.VAppPropertySpec; import com.vmware.vim25.VMwareDVSPortSetting; -import com.vmware.vim25.VStorageObject; -import com.vmware.vim25.VStorageObjectConfigInfo; import com.vmware.vim25.VimPortType; import com.vmware.vim25.VirtualDevice; import com.vmware.vim25.VirtualDeviceBackingInfo; diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java index 28f6bfcba28..943e0d81448 100644 --- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java @@ -2402,6 +2402,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q sb.and("clusterId", sb.entity().getClusterId(), SearchCriteria.Op.EQ); sb.and("hostAddress", sb.entity().getHostAddress(), SearchCriteria.Op.EQ); sb.and("scope", sb.entity().getScope(), SearchCriteria.Op.EQ); + sb.and("parent", sb.entity().getParent(), Op.EQ); SearchCriteria sc = sb.create(); @@ -2439,6 +2440,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q if (scopeType != null) { sc.setParameters("scope", scopeType.toString()); } + sc.setParameters("parent", 0); // search Pool details by ids Pair, Integer> uniquePoolPair = _poolJoinDao.searchAndCount(sc, searchFilter); diff --git a/server/src/main/java/com/cloud/api/query/vo/StoragePoolJoinVO.java b/server/src/main/java/com/cloud/api/query/vo/StoragePoolJoinVO.java index 565e290bd70..1831aaafac9 100644 --- a/server/src/main/java/com/cloud/api/query/vo/StoragePoolJoinVO.java +++ b/server/src/main/java/com/cloud/api/query/vo/StoragePoolJoinVO.java @@ -139,6 +139,9 @@ public class StoragePoolJoinVO extends BaseViewVO implements InternalIdentity, I @Column(name = "storage_provider_name") private String storageProviderName; + @Column(name = "parent") + private Long parent; + /** * @return the scope */ @@ -263,4 +266,8 @@ public class StoragePoolJoinVO extends BaseViewVO implements InternalIdentity, I public String getStorageProviderName() { return storageProviderName; } + + public Long getParent() { + return parent; + } } From 75fb1d91eee6e18b2247d39ea4a3cfda04101c07 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Wed, 22 Jul 2020 12:37:06 +0530 Subject: [PATCH 032/164] Fix adding Datastore clusters and listing --- .../storage/datastore/db/StoragePoolVO.java | 2 +- .../volume/datastore/PrimaryDataStoreHelper.java | 14 ++++++++++++++ .../datastore/provider/DefaultHostListener.java | 4 +++- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/StoragePoolVO.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/StoragePoolVO.java index 2cdcae54de2..c7eb72b469e 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/StoragePoolVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/StoragePoolVO.java @@ -120,7 +120,7 @@ public class StoragePoolVO implements StoragePool { private HypervisorType hypervisor; @Column(name = "parent") - private Long parent; + private Long parent = 0L; @Override public long getId() { diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/volume/datastore/PrimaryDataStoreHelper.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/volume/datastore/PrimaryDataStoreHelper.java index 31b57080aec..ac2c93567c3 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/volume/datastore/PrimaryDataStoreHelper.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/volume/datastore/PrimaryDataStoreHelper.java @@ -161,6 +161,13 @@ public class PrimaryDataStoreHelper { pool.setScope(ScopeType.CLUSTER); pool.setStatus(StoragePoolStatus.Up); this.dataStoreDao.update(pool.getId(), pool); + if(pool.getPoolType() == StoragePoolType.DatastoreCluster && pool.getParent() == 0) { + List childDatastores = dataStoreDao.listChildStoragePoolsInDatastoreCluster(pool.getId()); + for (StoragePoolVO child : childDatastores) { + child.setScope(ScopeType.CLUSTER); + this.dataStoreDao.update(child.getId(), child); + } + } return dataStoreMgr.getDataStore(store.getId(), DataStoreRole.Primary); } @@ -178,6 +185,13 @@ public class PrimaryDataStoreHelper { pool.setHypervisor(hypervisor); pool.setStatus(StoragePoolStatus.Up); this.dataStoreDao.update(pool.getId(), pool); + if(pool.getPoolType() == StoragePoolType.DatastoreCluster && pool.getParent() == 0) { + List childDatastores = dataStoreDao.listChildStoragePoolsInDatastoreCluster(pool.getId()); + for (StoragePoolVO child : childDatastores) { + child.setScope(ScopeType.ZONE); + this.dataStoreDao.update(child.getId(), child); + } + } return dataStoreMgr.getDataStore(store.getId(), DataStoreRole.Primary); } diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java index dac6514cc84..665b2631fb2 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java @@ -26,6 +26,7 @@ import com.cloud.agent.api.StoragePoolInfo; import com.cloud.alert.AlertManager; import com.cloud.exception.StorageConflictException; import com.cloud.storage.DataStoreRole; +import com.cloud.storage.Storage; import com.cloud.storage.StoragePool; import com.cloud.storage.StoragePoolHostVO; import com.cloud.storage.StoragePoolStatus; @@ -111,7 +112,7 @@ public class DefaultHostListener implements HypervisorHostListener { dataStoreVO = new StoragePoolVO(); dataStoreVO.setStorageProviderName(poolVO.getStorageProviderName()); dataStoreVO.setHostAddress(childStoragePoolInfo.getHost()); - dataStoreVO.setPoolType(poolVO.getPoolType()); + dataStoreVO.setPoolType(Storage.StoragePoolType.PreSetup); dataStoreVO.setPath(childStoragePoolInfo.getHostPath()); dataStoreVO.setPort(poolVO.getPort()); dataStoreVO.setName(childStoragePoolInfo.getName()); @@ -127,6 +128,7 @@ public class DefaultHostListener implements HypervisorHostListener { dataStoreVO.setUsedBytes(childDataStoreAnswer.getPoolInfo().getCapacityBytes() - childDataStoreAnswer.getPoolInfo().getAvailableBytes()); dataStoreVO.setHypervisor(poolVO.getHypervisor()); dataStoreVO.setScope(poolVO.getScope()); + dataStoreVO.setParent(poolVO.getId()); Map details = new HashMap<>(); if(StringUtils.isNotEmpty(childDataStoreAnswer.getPoolType())) { From 873f9dd9ac629bd350068401470169d177aa3328 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Thu, 23 Jul 2020 00:38:37 +0530 Subject: [PATCH 033/164] Datastore Clusters operations on putting into maintenance mode, update storage pool with tags, cancelling mantenance mode and deleting storage pool --- .../provider/DefaultHostListener.java | 1 - .../com/cloud/storage/StorageManagerImpl.java | 53 +++++++++++++++++-- .../storage/StoragePoolAutomationImpl.java | 8 +-- 3 files changed, 55 insertions(+), 7 deletions(-) diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java index 665b2631fb2..5c55a36c5b5 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java @@ -136,7 +136,6 @@ public class DefaultHostListener implements HypervisorHostListener { } List storageTags = storagePoolTagsDao.getStoragePoolTags(poolId); - storageTags.add("DataStoreCluster-" + poolVO.getUuid()); primaryStoreDao.persist(dataStoreVO, details, storageTags); updateStoragePoolHostVOAndDetails(dataStoreVO, hostId, childDataStoreAnswer); diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index b8a2f83d822..bc94dfc2a8f 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -822,6 +822,12 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C if (s_logger.isDebugEnabled()) { s_logger.debug("Updating Storage Pool Tags to :" + storagePoolTags); } + if (pool.getPoolType() == StoragePoolType.DatastoreCluster) { + List childStoragePools = _storagePoolDao.listChildStoragePoolsInDatastoreCluster(pool.getId()); + for (StoragePoolVO childPool : childStoragePools) { + _storagePoolTagsDao.persist(childPool.getId(), storagePoolTags); + } + } _storagePoolTagsDao.persist(pool.getId(), storagePoolTags); } @@ -918,10 +924,23 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C s_logger.warn("Unable to delete storage id: " + id + " due to it is not in Maintenance state"); throw new InvalidParameterValueException("Unable to delete storage due to it is not in Maintenance state, id: " + id); } - Pair vlms = _volsDao.getCountAndTotalByPool(id); + + if (sPool.getPoolType() == StoragePoolType.DatastoreCluster) { + // FR41 yet to handle on failure of deletion of any of the child storage pool + List childStoragePools = _storagePoolDao.listChildStoragePoolsInDatastoreCluster(sPool.getId()); + for (StoragePoolVO childPool : childStoragePools) { + deleteDataStoreInternal(childPool, forced); + } + } + return deleteDataStoreInternal(sPool, forced); + + } + + private boolean deleteDataStoreInternal(StoragePoolVO sPool, boolean forced) { + Pair vlms = _volsDao.getCountAndTotalByPool(sPool.getId()); if (forced) { if (vlms.first() > 0) { - Pair nonDstrdVlms = _volsDao.getNonDestroyedCountAndTotalByPool(id); + Pair nonDstrdVlms = _volsDao.getNonDestroyedCountAndTotalByPool(sPool.getId()); if (nonDstrdVlms.first() > 0) { throw new CloudRuntimeException("Cannot delete pool " + sPool.getName() + " as there are associated " + "non-destroyed vols for this pool"); } @@ -957,7 +976,7 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C } _storagePoolDao.releaseFromLockTable(lock.getId()); - s_logger.trace("Released lock for storage pool " + id); + s_logger.trace("Released lock for storage pool " + sPool.getId()); DataStoreProvider storeProvider = _dataStoreProviderMgr.getDataStoreProvider(sPool.getStorageProviderName()); DataStoreLifeCycle lifeCycle = storeProvider.getDataStoreLifeCycle(); @@ -1486,6 +1505,26 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C DataStoreProvider provider = _dataStoreProviderMgr.getDataStoreProvider(primaryStorage.getStorageProviderName()); DataStoreLifeCycle lifeCycle = provider.getDataStoreLifeCycle(); DataStore store = _dataStoreMgr.getDataStore(primaryStorage.getId(), DataStoreRole.Primary); + + if (primaryStorage.getPoolType() == StoragePoolType.DatastoreCluster) { + // Before preparing the datastorecluster to maintenance mode, the storagepools in the datastore cluster needs to put in maintenance + List childDatastores = _storagePoolDao.listChildStoragePoolsInDatastoreCluster(primaryStorageId); + Transaction.execute(new TransactionCallbackNoReturn() { + @Override + public void doInTransactionWithoutResult(TransactionStatus status) { + for (StoragePoolVO childDatastore : childDatastores) { + // set the pool state to prepare for maintenance, so that VMs will not migrate to the storagepools in the same cluster + childDatastore.setStatus(StoragePoolStatus.PrepareForMaintenance); + _storagePoolDao.update(childDatastore.getId(), childDatastore); + } + } + }); + for (StoragePoolVO childDatastore : childDatastores) { + //FR41 need to handle when one of the primary stores is unable to put in maintenance mode + DataStore childStore = _dataStoreMgr.getDataStore(childDatastore.getId(), DataStoreRole.Primary); + lifeCycle.maintain(childStore); + } + } lifeCycle.maintain(store); return (PrimaryDataStoreInfo)_dataStoreMgr.getDataStore(primaryStorage.getId(), DataStoreRole.Primary); @@ -1513,6 +1552,14 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C DataStoreProvider provider = _dataStoreProviderMgr.getDataStoreProvider(primaryStorage.getStorageProviderName()); DataStoreLifeCycle lifeCycle = provider.getDataStoreLifeCycle(); DataStore store = _dataStoreMgr.getDataStore(primaryStorage.getId(), DataStoreRole.Primary); + if (primaryStorage.getPoolType() == StoragePoolType.DatastoreCluster) { + //FR41 need to handle when one of the primary stores is unable to cancel the maintenance mode + List childDatastores = _storagePoolDao.listChildStoragePoolsInDatastoreCluster(primaryStorageId); + for (StoragePoolVO childDatastore : childDatastores) { + DataStore childStore = _dataStoreMgr.getDataStore(childDatastore.getId(), DataStoreRole.Primary); + lifeCycle.cancelMaintain(childStore); + } + } lifeCycle.cancelMaintain(store); return (PrimaryDataStoreInfo)_dataStoreMgr.getDataStore(primaryStorage.getId(), DataStoreRole.Primary); diff --git a/server/src/main/java/com/cloud/storage/StoragePoolAutomationImpl.java b/server/src/main/java/com/cloud/storage/StoragePoolAutomationImpl.java index 4ffd7d80b0d..15a72590fc4 100644 --- a/server/src/main/java/com/cloud/storage/StoragePoolAutomationImpl.java +++ b/server/src/main/java/com/cloud/storage/StoragePoolAutomationImpl.java @@ -117,9 +117,11 @@ public class StoragePoolAutomationImpl implements StoragePoolAutomation { spes = primaryDataStoreDao.listBy(pool.getDataCenterId(), pool.getPodId(), pool.getClusterId(), ScopeType.CLUSTER); } for (StoragePoolVO sp : spes) { - if (sp.getStatus() == StoragePoolStatus.PrepareForMaintenance) { - throw new CloudRuntimeException("Only one storage pool in a cluster can be in PrepareForMaintenance mode, " + sp.getId() + - " is already in PrepareForMaintenance mode "); + if (sp.getParent() != pool.getParent()) { // If Datastore cluster is tried to prepare for maintenance then child storage pools are also kept in PrepareForMaintenance mode + if (sp.getStatus() == StoragePoolStatus.PrepareForMaintenance) { + throw new CloudRuntimeException("Only one storage pool in a cluster can be in PrepareForMaintenance mode, " + sp.getId() + + " is already in PrepareForMaintenance mode "); + } } } StoragePool storagePool = (StoragePool)store; From 0335713738af0d9522b0c6645c404b0301821166 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Thu, 23 Jul 2020 00:53:39 +0530 Subject: [PATCH 034/164] Fix NPE on one of the register disk usecases --- .../java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java index 0382f21df60..3db74f317a1 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java @@ -2595,13 +2595,13 @@ public class VirtualMachineMO extends BaseMO { VirtualDeviceBackingInfo backingInfo = ((VirtualDisk)device).getBacking(); if (backingInfo instanceof VirtualDiskFlatVer2BackingInfo) { VirtualDiskFlatVer2BackingInfo diskBackingInfo = (VirtualDiskFlatVer2BackingInfo)backingInfo; - + String diskBackingFileName = diskBackingInfo.getFileName(); while (diskBackingInfo != null) { String deviceBusName = getDeviceBusName(devices, device); builder.addDisk(deviceBusName, diskBackingInfo.getFileName()); diskBackingInfo = diskBackingInfo.getParent(); } - DatastoreFile dsBackingFile = new DatastoreFile(diskBackingInfo.getFileName()); + DatastoreFile dsBackingFile = new DatastoreFile(diskBackingFileName); registerVirtualDisk((VirtualDisk) device, dsBackingFile); } } From fef39264693cedd4e7d50bc6938bb7c6ec2825eb Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Thu, 23 Jul 2020 01:23:14 +0530 Subject: [PATCH 035/164] Datastore cluster: update tag to the child storagepools also --- .../cloud/storage/dao/StoragePoolDetailsDaoImpl.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolDetailsDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolDetailsDaoImpl.java index 1e74d25632b..0c39a8c581a 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolDetailsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolDetailsDaoImpl.java @@ -21,11 +21,19 @@ import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.ConfigKey.Scope; import org.apache.cloudstack.framework.config.ScopedConfigStorage; import org.apache.cloudstack.resourcedetail.ResourceDetailsDaoBase; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailVO; import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; + +import javax.inject.Inject; +import java.util.List; public class StoragePoolDetailsDaoImpl extends ResourceDetailsDaoBase implements StoragePoolDetailsDao, ScopedConfigStorage { + @Inject + PrimaryDataStoreDao _storagePoolDao; + public StoragePoolDetailsDaoImpl() { } @@ -42,6 +50,10 @@ public class StoragePoolDetailsDaoImpl extends ResourceDetailsDaoBase ChildPools = _storagePoolDao.listChildStoragePoolsInDatastoreCluster(resourceId); + for(StoragePoolVO childPool : ChildPools) { + super.addDetail(new StoragePoolDetailVO(childPool.getId(), key, value, display)); + } super.addDetail(new StoragePoolDetailVO(resourceId, key, value, display)); } } From ed7efdf4d214982c1f571707fbdbf9ea5064e232 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Thu, 23 Jul 2020 04:18:21 +0530 Subject: [PATCH 036/164] Generalise linked clone formats which are different for each datastore type --- .../resource/VmwareStorageLayoutHelper.java | 182 +++++++++++------- .../resource/VmwareStorageProcessor.java | 4 +- 2 files changed, 111 insertions(+), 75 deletions(-) diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageLayoutHelper.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageLayoutHelper.java index d10c63bb3dd..b3564885447 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageLayoutHelper.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageLayoutHelper.java @@ -16,8 +16,11 @@ // under the License. package com.cloud.storage.resource; +import java.util.Arrays; import java.util.List; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; import org.apache.log4j.Logger; import com.cloud.hypervisor.vmware.mo.DatacenterMO; @@ -32,32 +35,49 @@ import com.cloud.utils.Pair; * To provide helper methods to handle storage layout in one place * */ -public class VmwareStorageLayoutHelper { +public class VmwareStorageLayoutHelper implements Configurable { private static final Logger s_logger = Logger.getLogger(VmwareStorageLayoutHelper.class); + static final ConfigKey VsphereLinkedCloneExtensions = new ConfigKey("Hidden", String.class, + "vsphere.linked.clone.extensions", "delta.vmdk,sesparse.vmdk", + "Comma separated list of linked clone disk formats allowed to handle storage in VMware", true); + + public static String[] getVmdkFilePairDatastorePath(DatastoreMO dsMo, String vmName, String vmdkName, VmwareStorageLayoutType layoutType, boolean linkedVmdk) throws Exception { - String[] filePair = new String[2]; + int i = 0; + String[] vSphereLinkedCloneExtensions = VsphereLinkedCloneExtensions.value().trim().split("\\s*,\\s*"); + String[] fileNames; + if (linkedVmdk) + fileNames = new String[vSphereLinkedCloneExtensions.length + 1]; + else + fileNames = new String[2]; + switch (layoutType) { case VMWARE: assert (vmName != null && !vmName.isEmpty()); - filePair[0] = getVmwareDatastorePathFromVmdkFileName(dsMo, vmName, vmdkName + ".vmdk"); + fileNames[i] = getVmwareDatastorePathFromVmdkFileName(dsMo, vmName, vmdkName + ".vmdk"); - if (linkedVmdk) - filePair[1] = getVmwareDatastorePathFromVmdkFileName(dsMo, vmName, vmdkName + "-delta.vmdk"); + if (linkedVmdk) { + for (int j=0 ; j < vSphereLinkedCloneExtensions.length; j++) { + fileNames[++i] = getVmwareDatastorePathFromVmdkFileName(dsMo, vmName, String.format("%s-%s",vmdkName, vSphereLinkedCloneExtensions[j])); + } + } else - filePair[1] = getVmwareDatastorePathFromVmdkFileName(dsMo, vmName, vmdkName + "-flat.vmdk"); - return filePair; + fileNames[i+1] = getVmwareDatastorePathFromVmdkFileName(dsMo, vmName, vmdkName + "-flat.vmdk"); + return fileNames; case CLOUDSTACK_LEGACY: - filePair[0] = getDatastorePathBaseFolderFromVmdkFileName(dsMo, vmdkName + ".vmdk"); + fileNames[i] = getDatastorePathBaseFolderFromVmdkFileName(dsMo, vmdkName + ".vmdk"); - if (linkedVmdk) - filePair[1] = getDatastorePathBaseFolderFromVmdkFileName(dsMo, vmdkName + "-delta.vmdk"); - else - filePair[1] = getDatastorePathBaseFolderFromVmdkFileName(dsMo, vmdkName + "-flat.vmdk"); - return filePair; + if (linkedVmdk) { + for (int j=0 ; j < vSphereLinkedCloneExtensions.length; j++) { + fileNames[++i] = getDatastorePathBaseFolderFromVmdkFileName(dsMo, String.format("%s-%s",vmdkName, vSphereLinkedCloneExtensions[j])); + } + } else + fileNames[i+1] = getDatastorePathBaseFolderFromVmdkFileName(dsMo, vmdkName + "-flat.vmdk"); + return fileNames; default: assert (false); @@ -71,26 +91,37 @@ public class VmwareStorageLayoutHelper { public static String[] getVmdkFilePairManagedDatastorePath(DatastoreMO dsMo, String vmName, String vmdkName, VmwareStorageLayoutType layoutType, boolean linkedVmdk) throws Exception { - String[] filePair = new String[2]; + int i = 0; + String[] vSphereLinkedCloneExtensions = VsphereLinkedCloneExtensions.value().trim().split("\\s*,\\s*"); + String[] fileNames; + if (linkedVmdk) + fileNames = new String[vSphereLinkedCloneExtensions.length + 1]; + else + fileNames = new String[2]; + switch (layoutType) { case VMWARE: assert (vmName != null && !vmName.isEmpty()); - filePair[0] = getVmwareDatastorePathFromVmdkFileName(dsMo, vmName, vmdkName + ".vmdk"); + fileNames[i] = getVmwareDatastorePathFromVmdkFileName(dsMo, vmName, vmdkName + ".vmdk"); - if (linkedVmdk) - filePair[1] = getVmwareDatastorePathFromVmdkFileName(dsMo, vmName, vmdkName + "-delta.vmdk"); - else - filePair[1] = getVmwareDatastorePathFromVmdkFileName(dsMo, vmName, vmdkName + "-flat.vmdk"); - return filePair; + if (linkedVmdk) { + for (int j=0 ; j < vSphereLinkedCloneExtensions.length; j++) { + fileNames[++i] = getVmwareDatastorePathFromVmdkFileName(dsMo, vmName, String.format("%s-%s",vmdkName, vSphereLinkedCloneExtensions[j])); + } + } else + fileNames[i+1] = getVmwareDatastorePathFromVmdkFileName(dsMo, vmName, vmdkName + "-flat.vmdk"); + return fileNames; case CLOUDSTACK_LEGACY: - filePair[0] = getDeprecatedLegacyDatastorePathFromVmdkFileName(dsMo, vmdkName + ".vmdk"); + fileNames[i] = getDeprecatedLegacyDatastorePathFromVmdkFileName(dsMo, vmdkName + ".vmdk"); - if (linkedVmdk) - filePair[1] = getDeprecatedLegacyDatastorePathFromVmdkFileName(dsMo, vmdkName + "-delta.vmdk"); - else - filePair[1] = getDeprecatedLegacyDatastorePathFromVmdkFileName(dsMo, vmdkName + "-flat.vmdk"); - return filePair; + if (linkedVmdk) { + for (int j=0 ; j < vSphereLinkedCloneExtensions.length; j++) { + fileNames[++i] = getDeprecatedLegacyDatastorePathFromVmdkFileName(dsMo, String.format("%s-%s",vmdkName, vSphereLinkedCloneExtensions[j])); + } + } else + fileNames[i+1] = getDeprecatedLegacyDatastorePathFromVmdkFileName(dsMo, vmdkName + "-flat.vmdk"); + return fileNames; default: assert (false); @@ -156,16 +187,20 @@ public class VmwareStorageLayoutHelper { syncVolumeToRootFolder(dcMo, ds, vmdkName, vmName, excludeFolders); } - if (ds.fileExists(vmdkFullCloneModeLegacyPair[1])) { - s_logger.info("sync " + vmdkFullCloneModeLegacyPair[1] + "->" + vmdkFullCloneModePair[1]); + for (int i=1; i" + vmdkFullCloneModePair[i]); - ds.moveDatastoreFile(vmdkFullCloneModeLegacyPair[1], dcMo.getMor(), ds.getMor(), vmdkFullCloneModePair[1], dcMo.getMor(), true); + ds.moveDatastoreFile(vmdkFullCloneModeLegacyPair[i], dcMo.getMor(), ds.getMor(), vmdkFullCloneModePair[i], dcMo.getMor(), true); + } } - if (ds.fileExists(vmdkLinkedCloneModeLegacyPair[1])) { - s_logger.info("sync " + vmdkLinkedCloneModeLegacyPair[1] + "->" + vmdkLinkedCloneModePair[1]); + for (int i=1; i" + vmdkLinkedCloneModePair[i]); - ds.moveDatastoreFile(vmdkLinkedCloneModeLegacyPair[1], dcMo.getMor(), ds.getMor(), vmdkLinkedCloneModePair[1], dcMo.getMor(), true); + ds.moveDatastoreFile(vmdkLinkedCloneModeLegacyPair[i], dcMo.getMor(), ds.getMor(), vmdkLinkedCloneModePair[i], dcMo.getMor(), true); + } } if (ds.fileExists(vmdkLinkedCloneModeLegacyPair[0])) { @@ -192,20 +227,18 @@ public class VmwareStorageLayoutHelper { } DatastoreFile srcDsFile = new DatastoreFile(fileDsFullPath); - String companionFilePath = srcDsFile.getCompanionPath(vmdkName + "-flat.vmdk"); - if (ds.fileExists(companionFilePath)) { - String targetPath = getDatastorePathBaseFolderFromVmdkFileName(ds, vmdkName + "-flat.vmdk"); - s_logger.info("Fixup folder-synchronization. move " + companionFilePath + " -> " + targetPath); - ds.moveDatastoreFile(companionFilePath, dcMo.getMor(), ds.getMor(), targetPath, dcMo.getMor(), true); - } + List vSphereFileExtensions = Arrays.asList(VsphereLinkedCloneExtensions.value().trim().split("\\s*,\\s*")); + // add flat file format to the above list + vSphereFileExtensions.add("flat.vmdk"); + for (String linkedCloneExtension : vSphereFileExtensions) { + String companionFilePath = srcDsFile.getCompanionPath(String.format("%s-%s",vmdkName, linkedCloneExtension)); + if (ds.fileExists(companionFilePath)) { + String targetPath = getDatastorePathBaseFolderFromVmdkFileName(ds, String.format("%s-%s",vmdkName, linkedCloneExtension)); - companionFilePath = srcDsFile.getCompanionPath(vmdkName + "-delta.vmdk"); - if (ds.fileExists(companionFilePath)) { - String targetPath = getDatastorePathBaseFolderFromVmdkFileName(ds, vmdkName + "-delta.vmdk"); - - s_logger.info("Fixup folder-synchronization. move " + companionFilePath + " -> " + targetPath); - ds.moveDatastoreFile(companionFilePath, dcMo.getMor(), ds.getMor(), targetPath, dcMo.getMor(), true); + s_logger.info("Fixup folder-synchronization. move " + companionFilePath + " -> " + targetPath); + ds.moveDatastoreFile(companionFilePath, dcMo.getMor(), ds.getMor(), targetPath, dcMo.getMor(), true); + } } // move the identity VMDK file the last @@ -234,18 +267,16 @@ public class VmwareStorageLayoutHelper { s_logger.info("Move " + file.getPath() + " -> " + targetFile.getPath()); dsMo.moveDatastoreFile(file.getPath(), dcMo.getMor(), dsMo.getMor(), targetFile.getPath(), dcMo.getMor(), true); - String pairSrcFilePath = file.getCompanionPath(file.getFileBaseName() + "-flat.vmdk"); - String pairTargetFilePath = targetFile.getCompanionPath(file.getFileBaseName() + "-flat.vmdk"); - if (dsMo.fileExists(pairSrcFilePath)) { - s_logger.info("Move " + pairSrcFilePath + " -> " + pairTargetFilePath); - dsMo.moveDatastoreFile(pairSrcFilePath, dcMo.getMor(), dsMo.getMor(), pairTargetFilePath, dcMo.getMor(), true); - } - - pairSrcFilePath = file.getCompanionPath(file.getFileBaseName() + "-delta.vmdk"); - pairTargetFilePath = targetFile.getCompanionPath(file.getFileBaseName() + "-delta.vmdk"); - if (dsMo.fileExists(pairSrcFilePath)) { - s_logger.info("Move " + pairSrcFilePath + " -> " + pairTargetFilePath); - dsMo.moveDatastoreFile(pairSrcFilePath, dcMo.getMor(), dsMo.getMor(), pairTargetFilePath, dcMo.getMor(), true); + List vSphereFileExtensions = Arrays.asList(VsphereLinkedCloneExtensions.value().trim().split("\\s*,\\s*")); + // add flat file format to the above list + vSphereFileExtensions.add("flat.vmdk"); + for (String linkedCloneExtension : vSphereFileExtensions) { + String pairSrcFilePath = file.getCompanionPath(String.format("%s-%s", file.getFileBaseName(), linkedCloneExtension)); + String pairTargetFilePath = targetFile.getCompanionPath(String.format("%s-%s", file.getFileBaseName(), linkedCloneExtension)); + if (dsMo.fileExists(pairSrcFilePath)) { + s_logger.info("Move " + pairSrcFilePath + " -> " + pairTargetFilePath); + dsMo.moveDatastoreFile(pairSrcFilePath, dcMo.getMor(), dsMo.getMor(), pairTargetFilePath, dcMo.getMor(), true); + } } } } else { @@ -322,24 +353,17 @@ public class VmwareStorageLayoutHelper { s_logger.warn("Unable to locate VMDK file: " + fileName); } - fileName = volumeName + "-flat.vmdk"; - fileFullPath = getLegacyDatastorePathFromVmdkFileName(dsMo, fileName); - if (!dsMo.fileExists(fileFullPath)) - fileFullPath = dsMo.searchFileInSubFolders(fileName, false, excludeFolders); - if (fileFullPath != null) { - dsMo.deleteFile(fileFullPath, dcMo.getMor(), true, excludeFolders); - } else { - s_logger.warn("Unable to locate VMDK file: " + fileName); - } - - fileName = volumeName + "-delta.vmdk"; - fileFullPath = getLegacyDatastorePathFromVmdkFileName(dsMo, fileName); - if (!dsMo.fileExists(fileFullPath)) - fileFullPath = dsMo.searchFileInSubFolders(fileName, false, excludeFolders); - if (fileFullPath != null) { - dsMo.deleteFile(fileFullPath, dcMo.getMor(), true, excludeFolders); - } else { - s_logger.warn("Unable to locate VMDK file: " + fileName); + List vSphereFileExtensions = Arrays.asList(VsphereLinkedCloneExtensions.value().trim().split("\\s*,\\s*")); + vSphereFileExtensions.add("flat.vmdk"); + for (String linkedCloneExtension : vSphereFileExtensions) { + fileFullPath = getLegacyDatastorePathFromVmdkFileName(dsMo, String.format("%s-%s", volumeName, linkedCloneExtension)); + if (!dsMo.fileExists(fileFullPath)) + fileFullPath = dsMo.searchFileInSubFolders(String.format("%s-%s", volumeName, linkedCloneExtension), false, excludeFolders); + if (fileFullPath != null) { + dsMo.deleteFile(fileFullPath, dcMo.getMor(), true, excludeFolders); + } else { + s_logger.warn("Unable to locate VMDK file: " + String.format("%s-%s", volumeName, linkedCloneExtension)); + } } } @@ -364,4 +388,14 @@ public class VmwareStorageLayoutHelper { public static String getVmwareDatastorePathFromVmdkFileName(DatastoreMO dsMo, String vmName, String vmdkFileName) throws Exception { return String.format("[%s] %s/%s", dsMo.getName(), vmName, vmdkFileName); } + + @Override + public String getConfigComponentName() { + return VmwareStorageLayoutHelper.class.getSimpleName(); + } + + @Override + public ConfigKey[] getConfigKeys() { + return new ConfigKey[] {VsphereLinkedCloneExtensions}; + } } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java index 22d0a92aea6..e1c16589692 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -671,7 +671,9 @@ public class VmwareStorageProcessor implements StorageProcessor { managedStoragePoolRootVolumeName, VmwareStorageLayoutType.CLOUDSTACK_LEGACY, false); dsMo.moveDatastoreFile(vmwareLayoutFilePair[0], dcMo.getMor(), dsMo.getMor(), legacyCloudStackLayoutFilePair[0], dcMo.getMor(), true); - dsMo.moveDatastoreFile(vmwareLayoutFilePair[1], dcMo.getMor(), dsMo.getMor(), legacyCloudStackLayoutFilePair[1], dcMo.getMor(), true); + for (int i=1; i Date: Fri, 24 Jul 2020 12:44:49 +0530 Subject: [PATCH 037/164] Fix immutable list to add another entry to list --- .../cloud/storage/resource/VmwareStorageLayoutHelper.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageLayoutHelper.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageLayoutHelper.java index b3564885447..164082c2a73 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageLayoutHelper.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageLayoutHelper.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.storage.resource; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -228,7 +229,7 @@ public class VmwareStorageLayoutHelper implements Configurable { DatastoreFile srcDsFile = new DatastoreFile(fileDsFullPath); - List vSphereFileExtensions = Arrays.asList(VsphereLinkedCloneExtensions.value().trim().split("\\s*,\\s*")); + List vSphereFileExtensions = new ArrayList<>(Arrays.asList(VsphereLinkedCloneExtensions.value().trim().split("\\s*,\\s*"))); // add flat file format to the above list vSphereFileExtensions.add("flat.vmdk"); for (String linkedCloneExtension : vSphereFileExtensions) { @@ -267,7 +268,7 @@ public class VmwareStorageLayoutHelper implements Configurable { s_logger.info("Move " + file.getPath() + " -> " + targetFile.getPath()); dsMo.moveDatastoreFile(file.getPath(), dcMo.getMor(), dsMo.getMor(), targetFile.getPath(), dcMo.getMor(), true); - List vSphereFileExtensions = Arrays.asList(VsphereLinkedCloneExtensions.value().trim().split("\\s*,\\s*")); + List vSphereFileExtensions = new ArrayList<>(Arrays.asList(VsphereLinkedCloneExtensions.value().trim().split("\\s*,\\s*"))); // add flat file format to the above list vSphereFileExtensions.add("flat.vmdk"); for (String linkedCloneExtension : vSphereFileExtensions) { @@ -353,7 +354,7 @@ public class VmwareStorageLayoutHelper implements Configurable { s_logger.warn("Unable to locate VMDK file: " + fileName); } - List vSphereFileExtensions = Arrays.asList(VsphereLinkedCloneExtensions.value().trim().split("\\s*,\\s*")); + List vSphereFileExtensions = new ArrayList<>(Arrays.asList(VsphereLinkedCloneExtensions.value().trim().split("\\s*,\\s*"))); vSphereFileExtensions.add("flat.vmdk"); for (String linkedCloneExtension : vSphereFileExtensions) { fileFullPath = getLegacyDatastorePathFromVmdkFileName(dsMo, String.format("%s-%s", volumeName, linkedCloneExtension)); From 4178a7cd9057d76c0a3d763b7240a358e45ad550 Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Tue, 28 Jul 2020 13:25:35 +0530 Subject: [PATCH 038/164] inter-cluster migration support --- check_diff | 103 ++++++++++++++++++ .../cloud/vm/VirtualMachineManagerImpl.java | 2 +- .../vmware/resource/VmwareResource.java | 10 +- .../vmware/mo/VirtualMachineMO.java | 4 +- 4 files changed, 112 insertions(+), 7 deletions(-) create mode 100644 check_diff diff --git a/check_diff b/check_diff new file mode 100644 index 00000000000..78b27ecd755 --- /dev/null +++ b/check_diff @@ -0,0 +1,103 @@ +diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +index 47d3013f22..2dbf257fb2 100644 +--- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java ++++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +@@ -5551,11 +5551,11 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir + throw new InvalidParameterValueException("Unable to find the VM by id=" + vmId); + } + +- if (vm.getState() != State.Stopped) { +- InvalidParameterValueException ex = new InvalidParameterValueException("VM is not Stopped, unable to migrate the vm having the specified id"); +- ex.addProxyObject(vm.getUuid(), "vmId"); +- throw ex; +- } ++// if (vm.getState() != State.Stopped) { ++// InvalidParameterValueException ex = new InvalidParameterValueException("VM is not Stopped, unable to migrate the vm having the specified id"); ++// ex.addProxyObject(vm.getUuid(), "vmId"); ++// throw ex; ++// } + + if (vm.getType() != VirtualMachine.Type.User) { + // OffLineVmwareMigration: *WHY* ? +@@ -5634,14 +5634,14 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir + throw new InvalidParameterValueException("Unable to find the VM by id=" + vmId); + } + // business logic +- if (vm.getState() != State.Running) { +- if (s_logger.isDebugEnabled()) { +- s_logger.debug("VM is not Running, unable to migrate the vm " + vm); +- } +- InvalidParameterValueException ex = new InvalidParameterValueException("VM is not Running, unable to migrate the vm with specified id"); +- ex.addProxyObject(vm.getUuid(), "vmId"); +- throw ex; +- } ++// if (vm.getState() != State.Running) { ++// if (s_logger.isDebugEnabled()) { ++// s_logger.debug("VM is not Running, unable to migrate the vm " + vm); ++// } ++// InvalidParameterValueException ex = new InvalidParameterValueException("VM is not Running, unable to migrate the vm with specified id"); ++// ex.addProxyObject(vm.getUuid(), "vmId"); ++// throw ex; ++// } + + checkIfHostOfVMIsInPrepareForMaintenanceState(vm.getHostId(), vmId, "Migrate"); + +@@ -5987,15 +5987,15 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir + } + + // OfflineVmwareMigration: this would be it ;) if multiple paths exist: unify +- if (vm.getState() != State.Running) { +- // OfflineVmwareMigration: and not vmware +- if (s_logger.isDebugEnabled()) { +- s_logger.debug("VM is not Running, unable to migrate the vm " + vm); +- } +- CloudRuntimeException ex = new CloudRuntimeException("VM is not Running, unable to migrate the vm with" + " specified id"); +- ex.addProxyObject(vm.getUuid(), "vmId"); +- throw ex; +- } ++// if (vm.getState() != State.Running) { ++// // OfflineVmwareMigration: and not vmware ++// if (s_logger.isDebugEnabled()) { ++// s_logger.debug("VM is not Running, unable to migrate the vm " + vm); ++// } ++// CloudRuntimeException ex = new CloudRuntimeException("VM is not Running, unable to migrate the vm with" + " specified id"); ++// ex.addProxyObject(vm.getUuid(), "vmId"); ++// throw ex; ++// } + + if(serviceOfferingDetailsDao.findDetail(vm.getServiceOfferingId(), GPU.Keys.pciDevice.toString()) != null) { + throw new InvalidParameterValueException("Live Migration of GPU enabled VM is not supported"); +diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java +index 62ecc9a5d5..36e102cb20 100644 +--- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java ++++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java +@@ -34,8 +34,6 @@ import java.util.concurrent.ExecutorService; + import java.util.concurrent.Executors; + import java.util.concurrent.Future; + +-import com.vmware.vim25.VStorageObject; +-import com.vmware.vim25.VStorageObjectConfigInfo; + import org.apache.commons.collections.CollectionUtils; + import org.apache.log4j.Logger; + import org.apache.commons.lang.StringUtils; +@@ -2430,13 +2428,13 @@ public class VirtualMachineMO extends BaseMO { + String deviceNumbering = getDeviceBusName(devices, device); + + s_logger.info("Disk backing : " + diskBackingInfo.getFileName() + " matches ==> " + deviceNumbering); +- if (((VirtualDisk) device).getVDiskId() == null) { +- s_logger.debug("vDiskid does not exist for volume " + vmdkDatastorePath + " registering the disk now"); +- VirtualStorageObjectManagerMO vStorageObjectManagerMO = new VirtualStorageObjectManagerMO(getOwnerDatacenter().first().getContext()); +- VStorageObject vStorageObject = vStorageObjectManagerMO.registerVirtualDisk(dsBackingFile, null, getOwnerDatacenter().first().getName()); +- VStorageObjectConfigInfo diskConfigInfo = vStorageObject.getConfig(); +- ((VirtualDisk) device).setVDiskId(diskConfigInfo.getId()); +- } ++// if (((VirtualDisk) device).getVDiskId() == null) { ++// s_logger.debug("vDiskid does not exist for volume " + vmdkDatastorePath + " registering the disk now"); ++// VirtualStorageObjectManagerMO vStorageObjectManagerMO = new VirtualStorageObjectManagerMO(getOwnerDatacenter().first().getContext()); ++// VStorageObject vStorageObject = vStorageObjectManagerMO.registerVirtualDisk(dsBackingFile, null, getOwnerDatacenter().first().getName()); ++// VStorageObjectConfigInfo diskConfigInfo = vStorageObject.getConfig(); ++// ((VirtualDisk) device).setVDiskId(diskConfigInfo.getId()); ++// } + return new Pair<>((VirtualDisk)device, deviceNumbering); + } + diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 14b3078cb4c..380a06e6152 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -2468,7 +2468,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac throw new CloudRuntimeException("Unable to find the host to migrate from: " + srcHostId); } - if (fromHost.getClusterId().longValue() != dest.getCluster().getId()) { + if (fromHost.getClusterId().longValue() != dest.getCluster().getId() && vm.getHypervisorType() != HypervisorType.VMware) { final List volumes = _volsDao.findCreatedByInstance(vm.getId()); for (final VolumeVO volume : volumes) { if (!_storagePoolDao.findById(volume.getPoolId()).getScope().equals(ScopeType.ZONE)) { diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 577e1a6e0af..e3a42b8e23a 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -4403,7 +4403,8 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa s_logger.debug("Preparing spec for volume : " + volume.getName()); morDsAtTarget = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(tgtHyperHost, filerTo.getUuid()); - morDsAtSource = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(srcHyperHost, filerTo.getUuid()); + morDsAtSource = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(srcHyperHost, volume.getPoolUuid()); + if (morDsAtTarget == null) { String msg = "Unable to find the target datastore: " + filerTo.getUuid() + " on target host: " + tgtHyperHost.getHyperHostName() + " to execute MigrateWithStorageCommand"; @@ -4431,7 +4432,6 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa s_logger.debug("Mounted datastore " + tgtDsHost + ":/" + tgtDsPath + " on " + _hostName); } } - // If datastore is VMFS and target datastore is not mounted or accessible to source host then fail migration. if (filerTo.getType().equals(StoragePoolType.VMFS) || filerTo.getType().equals(StoragePoolType.PreSetup)) { if (morDsAtSource == null) { @@ -4452,6 +4452,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa if (volume.getType() == Volume.Type.ROOT) { relocateSpec.setDatastore(morTgtDatastore); } + diskLocator = new VirtualMachineRelocateSpecDiskLocator(); diskLocator.setDatastore(morDsAtSource); Pair diskInfo = getVirtualDiskInfo(vmMo, appendFileType(volume.getPath(), VMDK_EXTENSION)); @@ -4476,8 +4477,9 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa diskLocators.add(diskLocator); } } - - relocateSpec.getDisk().addAll(diskLocators); + if (srcHyperHost.getHyperHostCluster().equals(tgtHyperHost.getHyperHostCluster())) { + relocateSpec.getDisk().addAll(diskLocators); + } // Prepare network at target before migration NicTO[] nics = vmTo.getNics(); diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java index 3db74f317a1..fc0b5ea8d1b 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java @@ -34,8 +34,6 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; -import com.vmware.vim25.VStorageObject; -import com.vmware.vim25.VStorageObjectConfigInfo; import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; import org.apache.commons.lang.StringUtils; @@ -63,6 +61,8 @@ import com.vmware.vim25.ParaVirtualSCSIController; import com.vmware.vim25.PropertyFilterSpec; import com.vmware.vim25.PropertySpec; import com.vmware.vim25.TraversalSpec; +import com.vmware.vim25.VStorageObject; +import com.vmware.vim25.VStorageObjectConfigInfo; import com.vmware.vim25.VirtualBusLogicController; import com.vmware.vim25.VirtualCdrom; import com.vmware.vim25.VirtualCdromIsoBackingInfo; From c8a28f34ee39bb2985c5ace6f96f36bda51ee1f8 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Mon, 27 Jul 2020 23:13:51 +0530 Subject: [PATCH 039/164] Use clonevm task while creating template from volume Old way is by taking snapshot of entire VM and then clone from that snapshot with the required disk --- .../resource/VmwareStorageProcessor.java | 10 ++++- .../vmware/mo/VirtualMachineMO.java | 38 +++++++++++++++++++ .../hypervisor/vmware/util/VmwareHelper.java | 6 +-- 3 files changed, 49 insertions(+), 5 deletions(-) diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java index e1c16589692..466056dba6c 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -1131,6 +1131,14 @@ public class VmwareStorageProcessor implements StorageProcessor { throw new Exception(msg); } + VmwareContext context = hostService.getServiceContext(null); + VmwareHypervisorHost hyperHost = hostService.getHyperHost(context, null); + DatacenterMO dcMo = new DatacenterMO(context, hyperHost.getHyperHostDatacenter()); + ManagedObjectReference morPool = hyperHost.getHyperHostOwnerResourcePool(); + vmMo.createFullCloneWithSpecificDisk(templateUniqueName, dcMo.getVmFolder(), morPool, VmwareHelper.getDiskDeviceDatastore(volumeDeviceInfo.first()), volumeDeviceInfo); + clonedVm = dcMo.findVm(templateUniqueName); + + /* FR41 THIS IS OLD way of creating template using snapshot if (!vmMo.createSnapshot(templateUniqueName, "Temporary snapshot for template creation", false, false)) { String msg = "Unable to take snapshot for creating template from volume. volume path: " + volumePath; s_logger.error(msg); @@ -1143,7 +1151,7 @@ public class VmwareStorageProcessor implements StorageProcessor { Pair cloneResult = vmMo.cloneFromCurrentSnapshot(workerVmName, 0, 4, volumeDeviceInfo.second(), VmwareHelper.getDiskDeviceDatastore(volumeDeviceInfo.first()), hardwareVersion); clonedVm = cloneResult.first(); - + * */ clonedVm.exportVm(secondaryMountPoint + "/" + installPath, templateUniqueName, false, false); // Get VMDK filename diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java index fc0b5ea8d1b..fc2621921dc 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java @@ -729,6 +729,44 @@ public class VirtualMachineMO extends BaseMO { return false; } + public boolean createFullCloneWithSpecificDisk(String cloneName, ManagedObjectReference morFolder, ManagedObjectReference morResourcePool, ManagedObjectReference morDs, Pair volumeDeviceInfo) + throws Exception { + + assert (morFolder != null); + assert (morResourcePool != null); + assert (morDs != null); + VirtualDisk requiredDisk = volumeDeviceInfo.first(); + + VirtualMachineRelocateSpec rSpec = new VirtualMachineRelocateSpec(); + List diskLocator = new ArrayList(1); + VirtualMachineRelocateSpecDiskLocator loc = new VirtualMachineRelocateSpecDiskLocator(); + loc.setDatastore(morDs); + loc.setDiskId(requiredDisk.getKey()); + loc.setDiskMoveType(VirtualMachineRelocateDiskMoveOptions.MOVE_ALL_DISK_BACKINGS_AND_DISALLOW_SHARING.value()); + diskLocator.add(loc); + + rSpec.setDiskMoveType(VirtualMachineRelocateDiskMoveOptions.MOVE_ALL_DISK_BACKINGS_AND_DISALLOW_SHARING.value()); + rSpec.getDisk().addAll(diskLocator); + rSpec.setPool(morResourcePool); + + VirtualMachineCloneSpec cloneSpec = new VirtualMachineCloneSpec(); + cloneSpec.setPowerOn(false); + cloneSpec.setTemplate(false); + cloneSpec.setLocation(rSpec); + + ManagedObjectReference morTask = _context.getService().cloneVMTask(_mor, morFolder, cloneName, cloneSpec); + + boolean result = _context.getVimClient().waitForTask(morTask); + if (result) { + _context.waitForTaskProgressDone(morTask); + return true; + } else { + s_logger.error("VMware cloneVM_Task failed due to " + TaskMO.getTaskFailureInfo(_context, morTask)); + } + + return false; + } + public boolean createFullClone(String cloneName, ManagedObjectReference morFolder, ManagedObjectReference morResourcePool, ManagedObjectReference morDs) throws Exception { diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java index 08b5672760a..babb954eec7 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java @@ -25,6 +25,7 @@ import java.io.PrintWriter; import java.io.StringWriter; import java.lang.reflect.Method; import java.util.ArrayList; +import java.util.Arrays; import java.util.Date; import java.util.GregorianCalendar; import java.util.List; @@ -249,10 +250,7 @@ public class VmwareHelper { backingInfo.setFileName(vmdkDatastorePathChain[0]); if (vmdkDatastorePathChain.length > 1) { - String[] parentDisks = new String[vmdkDatastorePathChain.length - 1]; - for (int i = 0; i < vmdkDatastorePathChain.length - 1; i++) - parentDisks[i] = vmdkDatastorePathChain[i + 1]; - + String[] parentDisks = Arrays.copyOfRange(vmdkDatastorePathChain, 1, vmdkDatastorePathChain.length); setParentBackingInfo(backingInfo, morDs, parentDisks); } From 4b10c6116763bed6f853a2bfdfbfbb4c60c81696 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Tue, 28 Jul 2020 09:08:59 +0530 Subject: [PATCH 040/164] Datastore cluster maintenance failure case handling. --- .../com/cloud/storage/StorageManagerImpl.java | 51 ++++++++++++------- 1 file changed, 34 insertions(+), 17 deletions(-) diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index bc94dfc2a8f..2381e8eec26 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -1507,29 +1507,46 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C DataStore store = _dataStoreMgr.getDataStore(primaryStorage.getId(), DataStoreRole.Primary); if (primaryStorage.getPoolType() == StoragePoolType.DatastoreCluster) { - // Before preparing the datastorecluster to maintenance mode, the storagepools in the datastore cluster needs to put in maintenance - List childDatastores = _storagePoolDao.listChildStoragePoolsInDatastoreCluster(primaryStorageId); - Transaction.execute(new TransactionCallbackNoReturn() { - @Override - public void doInTransactionWithoutResult(TransactionStatus status) { - for (StoragePoolVO childDatastore : childDatastores) { - // set the pool state to prepare for maintenance, so that VMs will not migrate to the storagepools in the same cluster - childDatastore.setStatus(StoragePoolStatus.PrepareForMaintenance); - _storagePoolDao.update(childDatastore.getId(), childDatastore); - } - } - }); - for (StoragePoolVO childDatastore : childDatastores) { - //FR41 need to handle when one of the primary stores is unable to put in maintenance mode - DataStore childStore = _dataStoreMgr.getDataStore(childDatastore.getId(), DataStoreRole.Primary); - lifeCycle.maintain(childStore); - } + handlePrepareDatastoreCluserMaintenance(lifeCycle, primaryStorageId); } lifeCycle.maintain(store); return (PrimaryDataStoreInfo)_dataStoreMgr.getDataStore(primaryStorage.getId(), DataStoreRole.Primary); } + private void handlePrepareDatastoreCluserMaintenance(DataStoreLifeCycle lifeCycle, Long primaryStorageId) { + // Before preparing the datastorecluster to maintenance mode, the storagepools in the datastore cluster needs to put in maintenance + List childDatastores = _storagePoolDao.listChildStoragePoolsInDatastoreCluster(primaryStorageId); + Transaction.execute(new TransactionCallbackNoReturn() { + @Override + public void doInTransactionWithoutResult(TransactionStatus status) { + for (StoragePoolVO childDatastore : childDatastores) { + // set the pool state to prepare for maintenance, so that VMs will not migrate to the storagepools in the same cluster + childDatastore.setStatus(StoragePoolStatus.PrepareForMaintenance); + _storagePoolDao.update(childDatastore.getId(), childDatastore); + } + } + }); + List maintenanceSuccessfulStoragePools = new ArrayList<>(); + for (StoragePoolVO childDatastore : childDatastores) { + //FR41 need to handle when one of the primary stores is unable to put in maintenance mode + DataStore childStore = _dataStoreMgr.getDataStore(childDatastore.getId(), DataStoreRole.Primary); + try { + lifeCycle.maintain(childStore); + } catch (Exception e) { + if (s_logger.isDebugEnabled()) { + s_logger.debug(String.format("Exception on maintenance preparation of one of the child datastores in datastore cluster %d with error %s", primaryStorageId, e)); + s_logger.debug(String.format("Cancelling the maintenance mode of child datastores in datastore cluster %d", primaryStorageId)); + } + for (DataStore dataStore: maintenanceSuccessfulStoragePools) { + lifeCycle.cancelMaintain(dataStore); + } + throw new CloudRuntimeException(String.format("Failed to prepare maintenance mode for datastore cluster %d with error %s %s", primaryStorageId, e.getMessage(), e)); + } + maintenanceSuccessfulStoragePools.add(childStore); + } + } + @Override @DB public PrimaryDataStoreInfo cancelPrimaryStorageForMaintenance(CancelPrimaryStorageMaintenanceCmd cmd) throws ResourceUnavailableException { From 7932f63390f7b4eef953bea0c6c878e3bc8b3034 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Tue, 28 Jul 2020 22:21:20 +0530 Subject: [PATCH 041/164] Add checks before actual deletion of datastore cluster --- .../com/cloud/storage/StorageManagerImpl.java | 37 +++++++++++++++++-- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index 2381e8eec26..0271e428c9b 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -927,13 +927,44 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C if (sPool.getPoolType() == StoragePoolType.DatastoreCluster) { // FR41 yet to handle on failure of deletion of any of the child storage pool - List childStoragePools = _storagePoolDao.listChildStoragePoolsInDatastoreCluster(sPool.getId()); - for (StoragePoolVO childPool : childStoragePools) { - deleteDataStoreInternal(childPool, forced); + if (checkIfDataStoreClusterCanbeDeleted(sPool, forced)) { + Transaction.execute(new TransactionCallbackNoReturn() { + @Override + public void doInTransactionWithoutResult(TransactionStatus status) { + List childStoragePools = _storagePoolDao.listChildStoragePoolsInDatastoreCluster(sPool.getId()); + for (StoragePoolVO childPool : childStoragePools) { + deleteDataStoreInternal(childPool, forced); + } + } + }); + } else { + throw new CloudRuntimeException("Cannot delete pool " + sPool.getName() + " as there are associated " + "non-destroyed vols for this pool"); } } return deleteDataStoreInternal(sPool, forced); + } + private boolean checkIfDataStoreClusterCanbeDeleted(StoragePoolVO sPool, boolean forced) { + List childStoragePools = _storagePoolDao.listChildStoragePoolsInDatastoreCluster(sPool.getId()); + boolean canDelete = true; + for (StoragePoolVO childPool : childStoragePools) { + Pair vlms = _volsDao.getCountAndTotalByPool(childPool.getId()); + if (forced) { + if (vlms.first() > 0) { + Pair nonDstrdVlms = _volsDao.getNonDestroyedCountAndTotalByPool(childPool.getId()); + if (nonDstrdVlms.first() > 0) { + canDelete = false; + break; + } + } + } else { + if (vlms.first() > 0) { + canDelete = false; + break; + } + } + } + return canDelete; } private boolean deleteDataStoreInternal(StoragePoolVO sPool, boolean forced) { From 812d42ebaaa62ce98efd27207c03ae73ac275fbc Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Tue, 28 Jul 2020 22:34:08 +0530 Subject: [PATCH 042/164] Fix updateconfiguration API to add settings to child datastores in case of datastore clusters --- .../com/cloud/configuration/ConfigurationManagerImpl.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java index e8c0284b14f..f4b37c33e7c 100755 --- a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java +++ b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java @@ -40,6 +40,7 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; import com.cloud.dc.dao.VsphereStoragePolicyDao; +import com.cloud.storage.Storage; import org.apache.cloudstack.acl.SecurityChecker; import org.apache.cloudstack.affinity.AffinityGroup; import org.apache.cloudstack.affinity.AffinityGroupService; @@ -602,6 +603,12 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati } _storagePoolDetailsDao.addDetail(resourceId, name, value, true); + if (pool.getPoolType() == Storage.StoragePoolType.DatastoreCluster) { + List childDataStores = _storagePoolDao.listChildStoragePoolsInDatastoreCluster(resourceId); + for (StoragePoolVO childDataStore: childDataStores) { + _storagePoolDetailsDao.addDetail(childDataStore.getId(), name, value, true); + } + } break; From e700ad72e7ac81ceaf1d8005a4fc7c30f398d9fb Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Tue, 28 Jul 2020 22:44:56 +0530 Subject: [PATCH 043/164] Fix prepare template API incase of Datastore cluster --- .../java/com/cloud/template/TemplateManagerImpl.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index d5e10383991..3a241faccd9 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -616,6 +616,18 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, private void prepareTemplateInOneStoragePool(final VMTemplateVO template, final StoragePoolVO pool) { s_logger.info("Schedule to preload template " + template.getId() + " into primary storage " + pool.getId()); + if (pool.getPoolType() == Storage.StoragePoolType.DatastoreCluster) { + List childDataStores = _poolDao.listChildStoragePoolsInDatastoreCluster(pool.getId()); + s_logger.debug("Schedule to preload template " + template.getId() + " into child datastores of DataStore cluster: " + pool.getId()); + for (StoragePoolVO childDataStore : childDataStores) { + prepareTemplateInOneStoragePoolInternal(template, childDataStore); + } + } else { + prepareTemplateInOneStoragePoolInternal(template, pool); + } + } + + private void prepareTemplateInOneStoragePoolInternal(final VMTemplateVO template, final StoragePoolVO pool) { _preloadExecutor.execute(new ManagedContextRunnable() { @Override protected void runInContext() { From fc05d3168f14584431a1a6b3c1f88cf04999cfaa Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Tue, 28 Jul 2020 23:19:07 +0530 Subject: [PATCH 044/164] Fix ListVMsCmd with storageid as datastore cluster --- .../com/cloud/api/query/QueryManagerImpl.java | 40 ++++++++++++------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java index 943e0d81448..850b2c64ca8 100644 --- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java @@ -31,6 +31,8 @@ import java.util.stream.Stream; import javax.inject.Inject; +import com.cloud.projects.dao.ProjectInvitationDao; +import com.cloud.user.dao.UserDao; import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.affinity.AffinityGroupDomainMapVO; import org.apache.cloudstack.affinity.AffinityGroupResponse; @@ -118,6 +120,8 @@ import org.apache.cloudstack.framework.config.Configurable; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.query.QueryService; import org.apache.cloudstack.resourcedetail.dao.DiskOfferingDetailsDao; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; @@ -417,6 +421,9 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q @Inject private TemplateDataStoreDao templateDataStoreDao; + @Inject + private PrimaryDataStoreDao _storagePoolDao; + @Inject private ProjectInvitationDao projectInvitationDao; @@ -973,7 +980,13 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q } if (storageId != null) { - sb.and("poolId", sb.entity().getPoolId(), SearchCriteria.Op.EQ); + StoragePoolVO poolVO = _storagePoolDao.findById((Long) storageId); + if (poolVO.getPoolType() == Storage.StoragePoolType.DatastoreCluster) { + List childDatastores = _storagePoolDao.listChildStoragePoolsInDatastoreCluster((Long) storageId); + sb.and("poolId", sb.entity().getPoolId(), SearchCriteria.Op.IN); + } else { + sb.and("poolId", sb.entity().getPoolId(), SearchCriteria.Op.EQ); + } } if (affinityGroupId != null) { @@ -1111,7 +1124,14 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q } if (storageId != null) { - sc.setParameters("poolId", storageId); + StoragePoolVO poolVO = _storagePoolDao.findById((Long) storageId); + if (poolVO.getPoolType() == Storage.StoragePoolType.DatastoreCluster) { + List childDatastores = _storagePoolDao.listChildStoragePoolsInDatastoreCluster((Long) storageId); + List childDatastoreIds = childDatastores.stream().map(mo -> mo.getId()).collect(Collectors.toList()); + sc.setParameters("poolId", childDatastoreIds.toArray()); + } else { + sc.setParameters("poolId", storageId); + } } } @@ -1504,19 +1524,15 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q } if (accountId != null) { - if (userId == null) { - sb.and().op("accountId", sb.entity().getAccountId(), SearchCriteria.Op.EQ); - sb.and("userIdNull", sb.entity().getUserId(), Op.NULL); - sb.cp(); - } else { - sb.and("accountId", sb.entity().getAccountId(), SearchCriteria.Op.EQ); - } + sb.and("accountId", sb.entity().getAccountId(), SearchCriteria.Op.EQ); } if (userId != null) { sb.and().op("userId", sb.entity().getUserId(), Op.EQ); sb.or("userIdNull", sb.entity().getUserId(), Op.NULL); sb.cp(); + } else { + sb.and("userIdNull", sb.entity().getUserId(), Op.NULL); } SearchCriteria sc = sb.create(); @@ -2562,7 +2578,6 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q Object keyword = cmd.getKeyword(); Long startIndex = cmd.getStartIndex(); Long pageSize = cmd.getPageSizeVal(); - Boolean readonly = cmd.getReadonly(); Filter searchFilter = new Filter(ImageStoreJoinVO.class, "id", Boolean.TRUE, startIndex, pageSize); @@ -2575,7 +2590,6 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q sb.and("protocol", sb.entity().getProtocol(), SearchCriteria.Op.EQ); sb.and("provider", sb.entity().getProviderName(), SearchCriteria.Op.EQ); sb.and("role", sb.entity().getRole(), SearchCriteria.Op.EQ); - sb.and("readonly", sb.entity().isReadonly(), Op.EQ); SearchCriteria sc = sb.create(); sc.setParameters("role", DataStoreRole.Image); @@ -2604,9 +2618,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q if (protocol != null) { sc.setParameters("protocol", protocol); } - if (readonly != null) { - sc.setParameters("readonly", readonly); - } + // search Store details by ids Pair, Integer> uniqueStorePair = _imageStoreJoinDao.searchAndCount(sc, searchFilter); Integer count = uniqueStorePair.second(); From 61dd85876b259be856568177cb7edb54f22a41fb Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Thu, 30 Jul 2020 10:06:37 +0530 Subject: [PATCH 045/164] Fix migrate vm and volume APIs in case if datastore cluster --- .../java/com/cloud/storage/StoragePool.java | 2 ++ .../main/java/com/cloud/vm/DiskProfile.java | 4 +++ .../service/VolumeOrchestrationService.java | 2 ++ .../api/storage/PrimaryDataStoreInfo.java | 2 ++ .../api/storage/StoragePoolAllocator.java | 2 ++ .../orchestration/VolumeOrchestrator.java | 28 +++++++++++++++++++ .../AbstractStoragePoolAllocator.java | 5 ++-- .../datastore/PrimaryDataStoreImpl.java | 5 ++++ .../cloud/server/ManagementServerImpl.java | 21 +++++++++++++- .../cloud/storage/VolumeApiServiceImpl.java | 8 ++++++ .../java/com/cloud/vm/UserVmManagerImpl.java | 8 ++++++ 11 files changed, 84 insertions(+), 3 deletions(-) diff --git a/api/src/main/java/com/cloud/storage/StoragePool.java b/api/src/main/java/com/cloud/storage/StoragePool.java index 3a2d3bd8fee..6c65ca17ac9 100644 --- a/api/src/main/java/com/cloud/storage/StoragePool.java +++ b/api/src/main/java/com/cloud/storage/StoragePool.java @@ -106,4 +106,6 @@ public interface StoragePool extends Identity, InternalIdentity { Hypervisor.HypervisorType getHypervisor(); boolean isManaged(); + + Long getParent(); } diff --git a/api/src/main/java/com/cloud/vm/DiskProfile.java b/api/src/main/java/com/cloud/vm/DiskProfile.java index 2b76c682ba3..175a92afaf9 100644 --- a/api/src/main/java/com/cloud/vm/DiskProfile.java +++ b/api/src/main/java/com/cloud/vm/DiskProfile.java @@ -100,6 +100,10 @@ public class DiskProfile { return name; } + public void setTags(String[] tags) { + this.tags = tags; + } + /** * @return tags for the disk. This can be used to match it to different storage pools. */ diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java index 9458de76353..db13c1f48a8 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java @@ -84,6 +84,8 @@ public interface VolumeOrchestrationService { String getVmNameOnVolume(Volume volume); + StoragePool findChildDataStoreInDataStoreCluster(DataCenter dc, Pod pod, Long clusterId, Long hostId, VirtualMachine vm, Long datastoreClusterId); + VolumeInfo createVolumeFromSnapshot(Volume volume, Snapshot snapshot, UserVm vm) throws StorageUnavailableException; Volume migrateVolume(Volume volume, StoragePool destPool) throws StorageUnavailableException; diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreInfo.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreInfo.java index 7f2f4dc6b85..3e072e8f1bb 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreInfo.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreInfo.java @@ -58,4 +58,6 @@ public interface PrimaryDataStoreInfo extends StoragePool { Map getDetails(); PrimaryDataStoreLifeCycle getLifeCycle(); + + Long getParent(); } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/StoragePoolAllocator.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/StoragePoolAllocator.java index dfdbd8ab92c..46f8f5e0429 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/StoragePoolAllocator.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/StoragePoolAllocator.java @@ -51,4 +51,6 @@ public interface StoragePoolAllocator extends Adapter { List allocateToPool(DiskProfile dskCh, VirtualMachineProfile vmProfile, DeploymentPlan plan, ExcludeList avoid, int returnUpTo); static int RETURN_UPTO_ALL = -1; + + List reorderPools(List pools, VirtualMachineProfile vmProfile, DeploymentPlan plan); } diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java index 3e68d3a4ab0..958771c561d 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java @@ -296,6 +296,34 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati return null; } + @Override + public StoragePool findChildDataStoreInDataStoreCluster(DataCenter dc, Pod pod, Long clusterId, Long hostId, VirtualMachine vm, Long datastoreClusterId) { + Long podId = null; + if (pod != null) { + podId = pod.getId(); + } else if (clusterId != null) { + Cluster cluster = _entityMgr.findById(Cluster.class, clusterId); + if (cluster != null) { + podId = cluster.getPodId(); + } + } + List childDatastores = _storagePoolDao.listChildStoragePoolsInDatastoreCluster(datastoreClusterId); + List suitablePools = new ArrayList(); + + for (StoragePoolVO childDatastore: childDatastores) + suitablePools.add((StoragePool)dataStoreMgr.getDataStore(childDatastore.getId(), DataStoreRole.Primary)); + + VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm); + for (StoragePoolAllocator allocator : _storagePoolAllocators) { + DataCenterDeployment plan = new DataCenterDeployment(dc.getId(), podId, clusterId, hostId, null, null); + final List poolList = allocator.reorderPools(suitablePools, profile, plan); + + if (poolList != null && !poolList.isEmpty()) { + return (StoragePool)dataStoreMgr.getDataStore(poolList.get(0).getId(), DataStoreRole.Primary); + } + } + return null; + } public Pair findPod(VirtualMachineTemplate template, ServiceOffering offering, DataCenter dc, long accountId, Set avoids) { for (PodAllocator allocator : _podAllocators) { final Pair pod = allocator.allocateTo(template, offering, dc, accountId, avoids); diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java index 68efe16fd7a..114385bb3c2 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java @@ -90,7 +90,7 @@ public abstract class AbstractStoragePoolAllocator extends AdapterBase implement @Override public List allocateToPool(DiskProfile dskCh, VirtualMachineProfile vmProfile, DeploymentPlan plan, ExcludeList avoid, int returnUpTo) { List pools = select(dskCh, vmProfile, plan, avoid, returnUpTo); - return reOrder(pools, vmProfile, plan); + return reorderPools(pools, vmProfile, plan); } protected List reorderPoolsByCapacity(DeploymentPlan plan, @@ -156,7 +156,8 @@ public abstract class AbstractStoragePoolAllocator extends AdapterBase implement return reorderedPools; } - protected List reOrder(List pools, VirtualMachineProfile vmProfile, DeploymentPlan plan) { + @Override + public List reorderPools(List pools, VirtualMachineProfile vmProfile, DeploymentPlan plan) { if (pools == null) { return null; } diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java index 81966784be0..16c3396884e 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java @@ -241,6 +241,11 @@ public class PrimaryDataStoreImpl implements PrimaryDataStore { return pdsv.isManaged(); } + @Override + public Long getParent() { + return pdsv.getParent(); + } + private boolean canCloneVolume() { return Boolean.valueOf(getDriver().getCapabilities().get(DataStoreCapabilities.CAN_CREATE_VOLUME_FROM_VOLUME.toString())); } diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index 80b54c09dfa..bed1ec00030 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -27,10 +27,13 @@ import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.TimeZone; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.function.Predicate; +import java.util.stream.Collectors; import javax.crypto.Mac; import javax.crypto.spec.SecretKeySpec; @@ -1482,10 +1485,26 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe } else { suitablePools = allPools; } - + abstractDataStoreClustersList((List) allPools); + abstractDataStoreClustersList((List) suitablePools); return new Pair, List>(allPools, suitablePools); } + private void abstractDataStoreClustersList(List storagePools) { + Predicate childDatastorePredicate = pool -> (pool.getParent() != 0); + List childDatastores = storagePools.stream().filter(childDatastorePredicate).collect(Collectors.toList()); + + if (!childDatastores.isEmpty()) { + storagePools.removeAll(childDatastores); + Set parentStoragePoolIds = childDatastores.stream().map(mo -> mo.getParent()).collect(Collectors.toSet()); + for (Long parentStoragePoolId : parentStoragePoolIds) { + StoragePool parentPool = _poolDao.findById(parentStoragePoolId); + if (!storagePools.contains(parentPool)) + storagePools.add(parentPool); + } + } + } + /** * This method looks for all storage pools that are compatible with the given volume. *
    diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index b5c33eb5fa1..e18c3553b01 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -30,6 +30,7 @@ import java.util.concurrent.ExecutionException; import javax.inject.Inject; +import com.cloud.dc.Pod; import org.apache.cloudstack.api.command.user.volume.AttachVolumeCmd; import org.apache.cloudstack.api.command.user.volume.CreateVolumeCmd; import org.apache.cloudstack.api.command.user.volume.DetachVolumeCmd; @@ -2192,6 +2193,13 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic throw new InvalidParameterValueException("Cannot migrate volume " + vol + "to the destination storage pool " + destPool.getName() + " as the storage pool is in maintenance mode."); } + if (destPool.getPoolType() == Storage.StoragePoolType.DatastoreCluster) { + DataCenter dc = _entityMgr.findById(DataCenter.class, vol.getDataCenterId()); + Pod destPoolPod = _entityMgr.findById(Pod.class, destPool.getPodId()); + + destPool = _volumeMgr.findChildDataStoreInDataStoreCluster(dc, destPoolPod, destPool.getClusterId(), null, null, destPool.getId()); + } + if (!storageMgr.storagePoolHasEnoughSpace(Collections.singletonList(vol), destPool)) { throw new CloudRuntimeException("Storage pool " + destPool.getName() + " does not have enough space to migrate volume " + vol.getName()); } diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index ef7617699ce..09dc4e7e3b7 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -504,6 +504,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir private ResourceTagDao resourceTagDao; @Inject private TemplateOVFPropertiesDao templateOVFPropertiesDao; + @Inject + private VolumeOrchestrationService _volumeMgr; private ScheduledExecutorService _executor = null; private ScheduledExecutorService _vmIpFetchExecutor = null; @@ -5556,6 +5558,12 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir } checkDestinationHypervisorType(destPool, vm); + if (destPool.getPoolType() == Storage.StoragePoolType.DatastoreCluster) { + DataCenter dc = _entityMgr.findById(DataCenter.class, vm.getDataCenterId()); + Pod destPoolPod = _entityMgr.findById(Pod.class, destPool.getPodId()); + + destPool = _volumeMgr.findChildDataStoreInDataStoreCluster(dc, destPoolPod, destPool.getClusterId(), null, null, destPool.getId()); + } _itMgr.storageMigration(vm.getUuid(), destPool); return _vmDao.findById(vm.getId()); From 1244fca367a6249bdb75b2f19c49196db2ca91cb Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Sat, 1 Aug 2020 21:33:37 +0530 Subject: [PATCH 046/164] Handle listsystemvms cmd in case of datastore clusters --- .../com/cloud/api/query/QueryManagerImpl.java | 1 - .../cloud/server/ManagementServerImpl.java | 25 ++++++++++++++++--- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java index 850b2c64ca8..0fd7aa1398d 100644 --- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java @@ -982,7 +982,6 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q if (storageId != null) { StoragePoolVO poolVO = _storagePoolDao.findById((Long) storageId); if (poolVO.getPoolType() == Storage.StoragePoolType.DatastoreCluster) { - List childDatastores = _storagePoolDao.listChildStoragePoolsInDatastoreCluster((Long) storageId); sb.and("poolId", sb.entity().getPoolId(), SearchCriteria.Op.IN); } else { sb.and("poolId", sb.entity().getPoolId(), SearchCriteria.Op.EQ); diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index bed1ec00030..f2eaeaf3f33 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -40,6 +40,7 @@ import javax.crypto.spec.SecretKeySpec; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.storage.Storage; import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.affinity.AffinityGroupProcessor; import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao; @@ -838,6 +839,8 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe private KeystoreManager _ksMgr; @Inject private DpdkHelper dpdkHelper; + @Inject + private PrimaryDataStoreDao _primaryDataStoreDao; private LockMasterListener _lockMasterListener; private final ScheduledExecutorService _eventExecutor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("EventChecker")); @@ -3323,9 +3326,16 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe sb.and("nulltype", sb.entity().getType(), SearchCriteria.Op.IN); if (storageId != null) { - final SearchBuilder volumeSearch = _volumeDao.createSearchBuilder(); - volumeSearch.and("poolId", volumeSearch.entity().getPoolId(), SearchCriteria.Op.EQ); - sb.join("volumeSearch", volumeSearch, sb.entity().getId(), volumeSearch.entity().getInstanceId(), JoinBuilder.JoinType.INNER); + StoragePoolVO storagePool = _primaryDataStoreDao.findById(storageId); + if (storagePool.getPoolType() == Storage.StoragePoolType.DatastoreCluster) { + final SearchBuilder volumeSearch = _volumeDao.createSearchBuilder(); + volumeSearch.and("poolId", volumeSearch.entity().getPoolId(), SearchCriteria.Op.IN); + sb.join("volumeSearch", volumeSearch, sb.entity().getId(), volumeSearch.entity().getInstanceId(), JoinBuilder.JoinType.INNER); + } else { + final SearchBuilder volumeSearch = _volumeDao.createSearchBuilder(); + volumeSearch.and("poolId", volumeSearch.entity().getPoolId(), SearchCriteria.Op.EQ); + sb.join("volumeSearch", volumeSearch, sb.entity().getId(), volumeSearch.entity().getInstanceId(), JoinBuilder.JoinType.INNER); + } } final SearchCriteria sc = sb.create(); @@ -3365,7 +3375,14 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe } if (storageId != null) { - sc.setJoinParameters("volumeSearch", "poolId", storageId); + StoragePoolVO storagePool = _primaryDataStoreDao.findById(storageId); + if (storagePool.getPoolType() == Storage.StoragePoolType.DatastoreCluster) { + List childDataStores = _primaryDataStoreDao.listChildStoragePoolsInDatastoreCluster(storageId); + List childDatastoreIds = childDataStores.stream().map(mo -> mo.getId()).collect(Collectors.toList()); + sc.setJoinParameters("volumeSearch", "poolId", childDatastoreIds.toArray()); + } else { + sc.setJoinParameters("volumeSearch", "poolId", storageId); + } } final Pair, Integer> result = _vmInstanceDao.searchAndCount(sc, searchFilter); From 48dd0dae0795f8800508051857fb794a508a3b4a Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Sun, 2 Aug 2020 00:26:58 +0530 Subject: [PATCH 047/164] Adding Storagepolicies to the service offering for root disks --- .../admin/offering/CreateServiceOfferingCmd.java | 8 ++++++++ .../service/dao/ServiceOfferingDetailsDao.java | 1 + .../dao/ServiceOfferingDetailsDaoImpl.java | 9 +++++++++ .../configuration/ConfigurationManagerImpl.java | 15 +++++++++++++-- .../com/cloud/storage/StorageManagerImpl.java | 14 +++++++++++++- 5 files changed, 44 insertions(+), 3 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateServiceOfferingCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateServiceOfferingCmd.java index 5015f7c51b8..82a0fcbd5b9 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateServiceOfferingCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateServiceOfferingCmd.java @@ -31,6 +31,7 @@ import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.response.DomainResponse; import org.apache.cloudstack.api.response.ServiceOfferingResponse; +import org.apache.cloudstack.api.response.VsphereStoragePoliciesResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.commons.collections.MapUtils; import org.apache.commons.collections.CollectionUtils; @@ -216,6 +217,9 @@ public class CreateServiceOfferingCmd extends BaseCmd { since = "4.13") private Integer minMemory; + @Parameter(name = ApiConstants.STORAGE_POLICY, type = CommandType.UUID, entityType = VsphereStoragePoliciesResponse.class,required = false, description = "Name of the storage policy defined at vCenter, this is applicable only for VMware") + private Long storagePolicy; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -418,6 +422,10 @@ public class CreateServiceOfferingCmd extends BaseCmd { return minMemory; } + public Long getStoragePolicy() { + return storagePolicy; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/engine/schema/src/main/java/com/cloud/service/dao/ServiceOfferingDetailsDao.java b/engine/schema/src/main/java/com/cloud/service/dao/ServiceOfferingDetailsDao.java index f905ab98dd5..10ba8d574a1 100644 --- a/engine/schema/src/main/java/com/cloud/service/dao/ServiceOfferingDetailsDao.java +++ b/engine/schema/src/main/java/com/cloud/service/dao/ServiceOfferingDetailsDao.java @@ -26,4 +26,5 @@ import com.cloud.utils.db.GenericDao; public interface ServiceOfferingDetailsDao extends GenericDao, ResourceDetailsDao { List findDomainIds(final long resourceId); List findZoneIds(final long resourceId); + String getDetail(Long diskOfferingId, String key); } \ No newline at end of file diff --git a/engine/schema/src/main/java/com/cloud/service/dao/ServiceOfferingDetailsDaoImpl.java b/engine/schema/src/main/java/com/cloud/service/dao/ServiceOfferingDetailsDaoImpl.java index 76840267461..0080174874d 100644 --- a/engine/schema/src/main/java/com/cloud/service/dao/ServiceOfferingDetailsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/service/dao/ServiceOfferingDetailsDaoImpl.java @@ -58,4 +58,13 @@ public class ServiceOfferingDetailsDaoImpl extends ResourceDetailsDaoBase filteredDomainIds = filterChildSubDomains(domainIds); @@ -2620,6 +2627,10 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati } } + if (storagePolicyID != null) { + detailsVO.add(new ServiceOfferingDetailsVO(offering.getId(), ApiConstants.STORAGE_POLICY, String.valueOf(storagePolicyID), false)); + } + if ((offering = _serviceOfferingDao.persist(offering)) != null) { for (Long domainId : filteredDomainIds) { detailsVO.add(new ServiceOfferingDetailsVO(offering.getId(), ApiConstants.DOMAIN_ID, String.valueOf(domainId), false)); diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index 0271e428c9b..eedcf71983b 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -42,6 +42,7 @@ import javax.inject.Inject; import com.cloud.agent.api.to.StorageFilerTO; import com.cloud.dc.VsphereStoragePolicyVO; import com.cloud.dc.dao.VsphereStoragePolicyDao; +import com.cloud.service.dao.ServiceOfferingDetailsDao; import com.cloud.utils.StringUtils; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.command.admin.storage.CancelPrimaryStorageMaintenanceCmd; @@ -307,6 +308,8 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C @Inject DiskOfferingDetailsDao _diskOfferingDetailsDao; @Inject + ServiceOfferingDetailsDao _serviceOfferingDetailsDao; + @Inject VsphereStoragePolicyDao _vsphereStoragePolicyDao; protected List _discoverers; @@ -1990,7 +1993,16 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C List> answers = new ArrayList>(); for (Volume volume : volumes) { - String storagePolicyId = _diskOfferingDetailsDao.getDetail(volume.getDiskOfferingId(), ApiConstants.STORAGE_POLICY); + String storagePolicyId = null; + if (volume.getVolumeType() == Type.ROOT) { + Long vmId = volume.getInstanceId(); + if (vmId != null) { + VMInstanceVO vm = _vmInstanceDao.findByIdIncludingRemoved(vmId); + storagePolicyId = _serviceOfferingDetailsDao.getDetail(vm.getServiceOfferingId(), ApiConstants.STORAGE_POLICY); + } + } else { + storagePolicyId = _diskOfferingDetailsDao.getDetail(volume.getDiskOfferingId(), ApiConstants.STORAGE_POLICY); + } if (org.apache.commons.lang.StringUtils.isNotEmpty(storagePolicyId)) { VsphereStoragePolicyVO storagePolicyVO = _vsphereStoragePolicyDao.findById(Long.parseLong(storagePolicyId)); List hostIds = getUpHostsInPool(pool.getId()); From 36d65905b1c732f20266de142ead6ae38736775a Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Tue, 4 Aug 2020 13:45:11 +0530 Subject: [PATCH 048/164] Base folder creation on datstores restrict only to primary storages, for secondary storage datastore it is not required --- .../vmware/manager/VmwareStorageManagerImpl.java | 2 +- .../cloud/hypervisor/vmware/resource/VmwareResource.java | 8 ++++---- .../cloud/storage/resource/VmwareStorageProcessor.java | 4 ++-- .../java/com/cloud/hypervisor/vmware/mo/ClusterMO.java | 4 ++-- .../main/java/com/cloud/hypervisor/vmware/mo/HostMO.java | 4 ++-- .../cloud/hypervisor/vmware/mo/VmwareHypervisorHost.java | 2 +- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareStorageManagerImpl.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareStorageManagerImpl.java index d59fcfbdf12..d8afb08f8df 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareStorageManagerImpl.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareStorageManagerImpl.java @@ -492,7 +492,7 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { ManagedObjectReference morDatastore = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, poolTO.getUuid()); if (morDatastore == null) { - morDatastore = hyperHost.mountDatastore(false, poolTO.getHost(), 0, poolTO.getPath(), poolTO.getUuid().replace("-", "")); + morDatastore = hyperHost.mountDatastore(false, poolTO.getHost(), 0, poolTO.getPath(), poolTO.getUuid().replace("-", ""), true); if (morDatastore == null) { throw new Exception("Unable to mount storage pool on host. storeUrl: " + poolTO.getHost() + ":/" + poolTO.getPath()); diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index e3a42b8e23a..977f8bab151 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -4424,7 +4424,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa // If datastore is NFS and target datastore is not already mounted on source host then mount the datastore. if (filerTo.getType().equals(StoragePoolType.NetworkFilesystem)) { if (morDsAtSource == null) { - morDsAtSource = srcHyperHost.mountDatastore(false, tgtDsHost, tgtDsPort, tgtDsPath, tgtDsName); + morDsAtSource = srcHyperHost.mountDatastore(false, tgtDsHost, tgtDsPort, tgtDsPath, tgtDsName, true); if (morDsAtSource == null) { throw new Exception("Unable to mount NFS datastore " + tgtDsHost + ":/" + tgtDsPath + " on " + _hostName); } @@ -4929,7 +4929,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa ManagedObjectReference morDatastore = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, pool.getUuid()); if (morDatastore == null) { - morDatastore = hyperHost.mountDatastore((pool.getType() == StoragePoolType.VMFS || pool.getType() == StoragePoolType.PreSetup), pool.getHost(), pool.getPort(), pool.getPath(), pool.getUuid().replace("-", "")); + morDatastore = hyperHost.mountDatastore((pool.getType() == StoragePoolType.VMFS || pool.getType() == StoragePoolType.PreSetup), pool.getHost(), pool.getPort(), pool.getPath(), pool.getUuid().replace("-", ""), true); } assert (morDatastore != null); @@ -5149,7 +5149,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa URI uri = new URI(storeUrl); VmwareHypervisorHost hyperHost = getHyperHost(getServiceContext()); - ManagedObjectReference morDatastore = hyperHost.mountDatastore(false, uri.getHost(), 0, uri.getPath(), storeName.replace("-", "")); + ManagedObjectReference morDatastore = hyperHost.mountDatastore(false, uri.getHost(), 0, uri.getPath(), storeName.replace("-", ""), false); if (morDatastore == null) throw new Exception("Unable to mount secondary storage on host. storeUrl: " + storeUrl); @@ -5161,7 +5161,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa String storeName = getSecondaryDatastoreUUID(storeUrl); URI uri = new URI(storeUrl); - ManagedObjectReference morDatastore = hyperHost.mountDatastore(false, uri.getHost(), 0, uri.getPath(), storeName.replace("-", "")); + ManagedObjectReference morDatastore = hyperHost.mountDatastore(false, uri.getHost(), 0, uri.getPath(), storeName.replace("-", ""), false); if (morDatastore == null) throw new Exception("Unable to mount secondary storage on host. storeUrl: " + storeUrl); diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java index 466056dba6c..96e5207615a 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -951,7 +951,7 @@ public class VmwareStorageProcessor implements StorageProcessor { if (morDatastore == null) { URI uri = new URI(destStore.getUrl()); - morDatastore = hyperHost.mountDatastore(false, uri.getHost(), 0, uri.getPath(), destStore.getUuid().replace("-", "")); + morDatastore = hyperHost.mountDatastore(false, uri.getHost(), 0, uri.getPath(), destStore.getUuid().replace("-", ""), true); if (morDatastore == null) { throw new Exception("Unable to mount storage pool on host. storeUrl: " + uri.getHost() + ":/" + uri.getPath()); @@ -2142,7 +2142,7 @@ public class VmwareStorageProcessor implements StorageProcessor { URI uri = new URI(storeUrl); VmwareHypervisorHost hyperHost = hostService.getHyperHost(hostService.getServiceContext(null), null); - ManagedObjectReference morDatastore = hyperHost.mountDatastore(false, uri.getHost(), 0, uri.getPath(), storeName.replace("-", "")); + ManagedObjectReference morDatastore = hyperHost.mountDatastore(false, uri.getHost(), 0, uri.getPath(), storeName.replace("-", ""), false); if (morDatastore == null) { throw new Exception("Unable to mount secondary storage on host. storeUrl: " + storeUrl); diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/ClusterMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/ClusterMO.java index 92b05f24521..e4926472824 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/ClusterMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/ClusterMO.java @@ -423,7 +423,7 @@ public class ClusterMO extends BaseMO implements VmwareHypervisorHost { } @Override - public ManagedObjectReference mountDatastore(boolean vmfsDatastore, String poolHostAddress, int poolHostPort, String poolPath, String poolUuid) throws Exception { + public ManagedObjectReference mountDatastore(boolean vmfsDatastore, String poolHostAddress, int poolHostPort, String poolPath, String poolUuid, boolean createBaseFolder) throws Exception { if (s_logger.isTraceEnabled()) s_logger.trace("vCenter API trace - mountDatastore(). target MOR: " + _mor.getValue() + ", vmfs: " + vmfsDatastore + ", poolHost: " + poolHostAddress + @@ -435,7 +435,7 @@ public class ClusterMO extends BaseMO implements VmwareHypervisorHost { if (hosts != null && hosts.size() > 0) { for (ManagedObjectReference morHost : hosts) { HostMO hostMo = new HostMO(_context, morHost); - morDs = hostMo.mountDatastore(vmfsDatastore, poolHostAddress, poolHostPort, poolPath, poolUuid); + morDs = hostMo.mountDatastore(vmfsDatastore, poolHostAddress, poolHostPort, poolPath, poolUuid, true); if (morDsFirst == null) morDsFirst = morDs; diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HostMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HostMO.java index 6554440a3e5..057c4f8f769 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HostMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HostMO.java @@ -836,7 +836,7 @@ public class HostMO extends BaseMO implements VmwareHypervisorHost { } @Override - public ManagedObjectReference mountDatastore(boolean vmfsDatastore, String poolHostAddress, int poolHostPort, String poolPath, String poolUuid) throws Exception { + public ManagedObjectReference mountDatastore(boolean vmfsDatastore, String poolHostAddress, int poolHostPort, String poolPath, String poolUuid, boolean createBaseFolder) throws Exception { if (s_logger.isTraceEnabled()) s_logger.trace("vCenter API trace - mountDatastore(). target MOR: " + _mor.getValue() + ", vmfs: " + vmfsDatastore + ", poolHost: " + poolHostAddress + @@ -889,7 +889,7 @@ public class HostMO extends BaseMO implements VmwareHypervisorHost { } } - if (dsMo != null && !"StoragePod".equals(morDatastore.getType())) { + if (dsMo != null && !"StoragePod".equals(morDatastore.getType()) && createBaseFolder) { HypervisorHostHelper.createBaseFolderInDatastore(dsMo, this); } diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VmwareHypervisorHost.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VmwareHypervisorHost.java index f99384a75e1..bfa9db7a19b 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VmwareHypervisorHost.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VmwareHypervisorHost.java @@ -71,7 +71,7 @@ public interface VmwareHypervisorHost { ObjectContent[] getDatastorePropertiesOnHyperHost(String[] propertyPaths) throws Exception; - ManagedObjectReference mountDatastore(boolean vmfsDatastore, String poolHostAddress, int poolHostPort, String poolPath, String poolUuid) throws Exception; + ManagedObjectReference mountDatastore(boolean vmfsDatastore, String poolHostAddress, int poolHostPort, String poolPath, String poolUuid, boolean createBaseFolder) throws Exception; void unmountDatastore(String poolUuid) throws Exception; From 1b18554794d1891539844ecc72e8c14aa47bdd93 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Tue, 4 Aug 2020 14:15:01 +0530 Subject: [PATCH 049/164] Fix test cases --- .../cloudstack/api/agent/test/BackupSnapshotCommandTest.java | 5 +++++ .../cloudstack/api/agent/test/CheckNetworkAnswerTest.java | 5 +++++ .../cloudstack/api/agent/test/SnapshotCommandTest.java | 5 +++++ 3 files changed, 15 insertions(+) diff --git a/core/src/test/java/org/apache/cloudstack/api/agent/test/BackupSnapshotCommandTest.java b/core/src/test/java/org/apache/cloudstack/api/agent/test/BackupSnapshotCommandTest.java index edc90aab770..4bdef109af3 100644 --- a/core/src/test/java/org/apache/cloudstack/api/agent/test/BackupSnapshotCommandTest.java +++ b/core/src/test/java/org/apache/cloudstack/api/agent/test/BackupSnapshotCommandTest.java @@ -137,6 +137,11 @@ public class BackupSnapshotCommandTest { @Override public boolean isManaged() { return false; } + @Override + public Long getParent() { + return 0L; + } + @Override public Long getPodId() { return 0L; diff --git a/core/src/test/java/org/apache/cloudstack/api/agent/test/CheckNetworkAnswerTest.java b/core/src/test/java/org/apache/cloudstack/api/agent/test/CheckNetworkAnswerTest.java index 4d49c99ee90..8b2550151f3 100644 --- a/core/src/test/java/org/apache/cloudstack/api/agent/test/CheckNetworkAnswerTest.java +++ b/core/src/test/java/org/apache/cloudstack/api/agent/test/CheckNetworkAnswerTest.java @@ -176,6 +176,11 @@ public class CheckNetworkAnswerTest { @Override public boolean isManaged() { return false; } + @Override + public Long getParent() { + return 0L; + } + @Override public Long getPodId() { return 0L; diff --git a/core/src/test/java/org/apache/cloudstack/api/agent/test/SnapshotCommandTest.java b/core/src/test/java/org/apache/cloudstack/api/agent/test/SnapshotCommandTest.java index 576419ab652..e134a94e1e5 100644 --- a/core/src/test/java/org/apache/cloudstack/api/agent/test/SnapshotCommandTest.java +++ b/core/src/test/java/org/apache/cloudstack/api/agent/test/SnapshotCommandTest.java @@ -138,6 +138,11 @@ public class SnapshotCommandTest { @Override public boolean isManaged() { return false; } + @Override + public Long getParent() { + return 0L; + } + @Override public Long getPodId() { return 0L; From e211b92324686a56c88b4cd3e02c7f50b502558b Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Tue, 4 Aug 2020 17:45:09 +0530 Subject: [PATCH 050/164] Changed workflow for storing volumes in sec storage --- .../admin/offering/CreateDiskOfferingCmd.java | 2 +- .../offering/CreateServiceOfferingCmd.java | 2 +- .../resource/VmwareStorageProcessor.java | 52 ++++++++----------- .../vmware/mo/VirtualMachineMO.java | 10 ++-- 4 files changed, 30 insertions(+), 36 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateDiskOfferingCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateDiskOfferingCmd.java index 6b9ddce75c2..a830777031f 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateDiskOfferingCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateDiskOfferingCmd.java @@ -152,7 +152,7 @@ public class CreateDiskOfferingCmd extends BaseCmd { since = "4.14") private String cacheMode; - @Parameter(name = ApiConstants.STORAGE_POLICY, type = CommandType.UUID, entityType = VsphereStoragePoliciesResponse.class,required = false, description = "Name of the storage policy defined at vCenter, this is applicable only for VMware") + @Parameter(name = ApiConstants.STORAGE_POLICY, type = CommandType.UUID, entityType = VsphereStoragePoliciesResponse.class,required = false, description = "Name of the storage policy defined at vCenter, this is applicable only for VMware", since = "4.15") private Long storagePolicy; ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateServiceOfferingCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateServiceOfferingCmd.java index 82a0fcbd5b9..745e6a0f3c6 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateServiceOfferingCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateServiceOfferingCmd.java @@ -217,7 +217,7 @@ public class CreateServiceOfferingCmd extends BaseCmd { since = "4.13") private Integer minMemory; - @Parameter(name = ApiConstants.STORAGE_POLICY, type = CommandType.UUID, entityType = VsphereStoragePoliciesResponse.class,required = false, description = "Name of the storage policy defined at vCenter, this is applicable only for VMware") + @Parameter(name = ApiConstants.STORAGE_POLICY, type = CommandType.UUID, entityType = VsphereStoragePoliciesResponse.class,required = false, description = "Name of the storage policy defined at vCenter, this is applicable only for VMware", since = "4.15") private Long storagePolicy; ///////////////////////////////////////////////////// diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java index 96e5207615a..cd6edbe964b 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -1015,11 +1015,9 @@ public class VmwareStorageProcessor implements StorageProcessor { workerVm.attachDisk(new String[] {datastoreVolumePath}, morDs); vmMo = workerVm; clonedWorkerVMNeeded = false; - } else { - vmMo.createSnapshot(exportName, "Temporary snapshot for copy-volume command", false, false); } - exportVolumeToSecondaryStorage(vmMo, volumePath, secStorageUrl, destVolumePath, exportName, hostService.getWorkerName(hyperHost.getContext(), cmd, 1), _nfsVersion, clonedWorkerVMNeeded); + exportVolumeToSecondaryStorage(hyperHost.getContext(), vmMo, hyperHost, volumePath, secStorageUrl, destVolumePath, exportName, hostService.getWorkerName(hyperHost.getContext(), cmd, 1), _nfsVersion, clonedWorkerVMNeeded); return new Pair<>(destVolumePath, exportName); } finally { @@ -1104,7 +1102,7 @@ public class VmwareStorageProcessor implements StorageProcessor { } } - private Ternary createTemplateFromVolume(VirtualMachineMO vmMo, String installPath, long templateId, String templateUniqueName, + private Ternary createTemplateFromVolume(VmwareContext context, VirtualMachineMO vmMo, VmwareHypervisorHost hyperHost, String installPath, long templateId, String templateUniqueName, String secStorageUrl, String volumePath, String workerVmName, String nfsVersion) throws Exception { String secondaryMountPoint = mountService.getMountPoint(secStorageUrl, nfsVersion); @@ -1131,8 +1129,6 @@ public class VmwareStorageProcessor implements StorageProcessor { throw new Exception(msg); } - VmwareContext context = hostService.getServiceContext(null); - VmwareHypervisorHost hyperHost = hostService.getHyperHost(context, null); DatacenterMO dcMo = new DatacenterMO(context, hyperHost.getHyperHostDatacenter()); ManagedObjectReference morPool = hyperHost.getHyperHostOwnerResourcePool(); vmMo.createFullCloneWithSpecificDisk(templateUniqueName, dcMo.getVmFolder(), morPool, VmwareHelper.getDiskDeviceDatastore(volumeDeviceInfo.first()), volumeDeviceInfo); @@ -1184,8 +1180,6 @@ public class VmwareStorageProcessor implements StorageProcessor { clonedVm.detachAllDisks(); clonedVm.destroy(); } - - vmMo.removeSnapshot(templateUniqueName, false); } } @@ -1211,7 +1205,7 @@ public class VmwareStorageProcessor implements StorageProcessor { if (volume.getVmName() == null) { ManagedObjectReference secMorDs = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, volume.getDataStore().getUuid()); DatastoreMO dsMo = new DatastoreMO(hyperHost.getContext(), secMorDs); - workerVmMo = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, "workervm"+volume.getUuid(), null); + workerVmMo = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, "workervm"+volume.getUuid()); if (workerVmMo == null) { throw new Exception("Unable to find created worker VM"); } @@ -1244,9 +1238,8 @@ public class VmwareStorageProcessor implements StorageProcessor { } Ternary result = - createTemplateFromVolume(vmMo, template.getPath(), template.getId(), template.getName(), secondaryStoragePoolURL, volumePath, + createTemplateFromVolume(context, vmMo, hyperHost, template.getPath(), template.getId(), template.getName(), secondaryStoragePoolURL, volumePath, hostService.getWorkerName(context, cmd, 0), _nfsVersion); - TemplateObjectTO newTemplate = new TemplateObjectTO(); newTemplate.setPath(result.first()); newTemplate.setFormat(ImageFormat.OVA); @@ -1568,7 +1561,7 @@ public class VmwareStorageProcessor implements StorageProcessor { ManagedObjectReference dsMor = hyperHost.findDatastoreByName(dsFile.getDatastoreName()); DatastoreMO dsMo = new DatastoreMO(context, dsMor); - VirtualMachineMO workerVM = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, workerVMName, null); + VirtualMachineMO workerVM = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, workerVMName); if (workerVM == null) { throw new CloudRuntimeException("Failed to find the newly created worker VM: " + workerVMName); @@ -1706,7 +1699,7 @@ public class VmwareStorageProcessor implements StorageProcessor { } // return Pair - private Pair exportVolumeToSecondaryStorage(VirtualMachineMO vmMo, String volumePath, String secStorageUrl, String secStorageDir, + private Pair exportVolumeToSecondaryStorage(VmwareContext context, VirtualMachineMO vmMo, VmwareHypervisorHost hyperHost, String volumePath, String secStorageUrl, String secStorageDir, String exportName, String workerVmName, String nfsVersion, boolean clonedWorkerVMNeeded) throws Exception { String secondaryMountPoint = mountService.getMountPoint(secStorageUrl, nfsVersion); @@ -1739,13 +1732,18 @@ public class VmwareStorageProcessor implements StorageProcessor { String disks[] = vmMo.getCurrentSnapshotDiskChainDatastorePaths(diskDevice); if (clonedWorkerVMNeeded) { // 4 MB is the minimum requirement for VM memory in VMware - Pair cloneResult = - vmMo.cloneFromCurrentSnapshot(workerVmName, 0, 4, diskDevice, VmwareHelper.getDiskDeviceDatastore(volumeDeviceInfo.first()), virtualHardwareVersion); - clonedVm = cloneResult.first(); - clonedVm.exportVm(exportPath, exportName, false, false); - } else { - vmMo.exportVm(exportPath, exportName, false, false); + DatacenterMO dcMo = new DatacenterMO(context, hyperHost.getHyperHostDatacenter()); + ManagedObjectReference morPool = hyperHost.getHyperHostOwnerResourcePool(); + vmMo.createFullCloneWithSpecificDisk(exportName, dcMo.getVmFolder(), morPool, VmwareHelper.getDiskDeviceDatastore(volumeDeviceInfo.first()), volumeDeviceInfo); + clonedVm = dcMo.findVm(exportName); + if (clonedVm == null) { + String msg = "Failed to clone VM. volume path: " + volumePath; + s_logger.error(msg); + throw new Exception(msg); + } } + vmMo.exportVm(exportPath, exportName, false, false); + return new Pair<>(diskDevice, disks); } finally { if (clonedVm != null) { @@ -1756,12 +1754,12 @@ public class VmwareStorageProcessor implements StorageProcessor { } // Ternary - private Ternary backupSnapshotToSecondaryStorage(VirtualMachineMO vmMo, String installPath, String volumePath, String snapshotUuid, + private Ternary backupSnapshotToSecondaryStorage(VmwareContext context, VirtualMachineMO vmMo, VmwareHypervisorHost hypervisorHost, String installPath, String volumePath, String snapshotUuid, String secStorageUrl, String prevSnapshotUuid, String prevBackupUuid, String workerVmName, String nfsVersion) throws Exception { String backupUuid = UUID.randomUUID().toString(); - Pair snapshotInfo = exportVolumeToSecondaryStorage(vmMo, volumePath, secStorageUrl, installPath, backupUuid, workerVmName, nfsVersion, true); + Pair snapshotInfo = exportVolumeToSecondaryStorage(context, vmMo, hypervisorHost, volumePath, secStorageUrl, installPath, backupUuid, workerVmName, nfsVersion, true); return new Ternary<>(backupUuid, snapshotInfo.first(), snapshotInfo.second()); } @@ -1830,12 +1828,8 @@ public class VmwareStorageProcessor implements StorageProcessor { hasOwnerVm = true; } - if (!vmMo.createSnapshot(snapshotUuid, "Snapshot taken for " + srcSnapshot.getName(), false, false)) { - throw new Exception("Failed to take snapshot " + srcSnapshot.getName() + " on vm: " + vmName); - } - backupResult = - backupSnapshotToSecondaryStorage(vmMo, destSnapshot.getPath(), srcSnapshot.getVolume().getPath(), snapshotUuid, secondaryStorageUrl, + backupSnapshotToSecondaryStorage(context, vmMo, hyperHost, destSnapshot.getPath(), srcSnapshot.getVolume().getPath(), snapshotUuid, secondaryStorageUrl, prevSnapshotUuid, prevBackupUuid, hostService.getWorkerName(context, cmd, 1), _nfsVersion); snapshotBackupUuid = backupResult.first(); @@ -1916,7 +1910,7 @@ public class VmwareStorageProcessor implements StorageProcessor { } } } else { - s_logger.error("Can not find the snapshot we just used ?!"); + s_logger.info("No snapshots created to be deleted!"); } } @@ -2281,7 +2275,7 @@ public class VmwareStorageProcessor implements StorageProcessor { * String dummyVmName = hostService.getWorkerName(context, cmd, 0); try { s_logger.info("Create worker VM " + dummyVmName); - vmMo = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, dummyVmName, null); + vmMo = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, dummyVmName); if (vmMo == null) { throw new Exception("Unable to create a dummy VM for volume creation"); } @@ -2966,7 +2960,7 @@ public class VmwareStorageProcessor implements StorageProcessor { String dummyVmName = hostService.getWorkerName(context, cmd, 0); - VirtualMachineMO vmMo = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, dummyVmName, null); + VirtualMachineMO vmMo = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, dummyVmName); if (vmMo == null) { throw new Exception("Unable to create a dummy VM for volume creation"); diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java index fc2621921dc..c4e1493a5f4 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java @@ -303,7 +303,7 @@ public class VirtualMachineMO extends BaseMO { try { Thread.sleep(1000); } catch (InterruptedException e) { - s_logger.debug("[ignored] interupted while powering of vm."); + s_logger.debug("[ignored] interrupted while powering of vm."); } } @@ -336,7 +336,7 @@ public class VirtualMachineMO extends BaseMO { try { Thread.sleep(1000); } catch (InterruptedException e) { - s_logger.debug("[ignored] interupted while powering of vm unconditionaly."); + s_logger.debug("[ignored] interrupted while powering of vm unconditionally."); } } return true; @@ -370,7 +370,7 @@ public class VirtualMachineMO extends BaseMO { try { Thread.sleep(1000); } catch (InterruptedException e) { - s_logger.debug("[ignored] interupted while pausing after power off."); + s_logger.debug("[ignored] interrupted while pausing after power off."); } } else { break; @@ -1780,7 +1780,7 @@ public class VirtualMachineMO extends BaseMO { command.add((new File(name).getName())); } - s_logger.info("Package OVA with commmand: " + command.toString()); + s_logger.info("Package OVA with command: " + command.toString()); command.execute(); // to be safe, physically test existence of the target OVA file @@ -1793,7 +1793,7 @@ public class VirtualMachineMO extends BaseMO { success = true; } } - s_logger.info("volss: copy vmdk and ovf file finishes " + System.currentTimeMillis()); + s_logger.info("volss: copy vmdk and ovf file finished " + System.currentTimeMillis()); } catch (Throwable e) { s_logger.error("Unexpected exception ", e); } finally { From 70e19b28e6b2d5d5fb0e66da85e46b81b3939071 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Wed, 5 Aug 2020 12:22:23 +0530 Subject: [PATCH 051/164] fix some more test cases --- server/src/main/java/com/cloud/vm/UserVmManagerImpl.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 09dc4e7e3b7..862190e3cb3 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -504,8 +504,6 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir private ResourceTagDao resourceTagDao; @Inject private TemplateOVFPropertiesDao templateOVFPropertiesDao; - @Inject - private VolumeOrchestrationService _volumeMgr; private ScheduledExecutorService _executor = null; private ScheduledExecutorService _vmIpFetchExecutor = null; @@ -5562,7 +5560,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir DataCenter dc = _entityMgr.findById(DataCenter.class, vm.getDataCenterId()); Pod destPoolPod = _entityMgr.findById(Pod.class, destPool.getPodId()); - destPool = _volumeMgr.findChildDataStoreInDataStoreCluster(dc, destPoolPod, destPool.getClusterId(), null, null, destPool.getId()); + destPool = volumeMgr.findChildDataStoreInDataStoreCluster(dc, destPoolPod, destPool.getClusterId(), null, null, destPool.getId()); } _itMgr.storageMigration(vm.getUuid(), destPool); From a5531a9e8612d9511bca5b551811dc274f16a1f9 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Thu, 6 Aug 2020 13:03:49 +0530 Subject: [PATCH 052/164] Changed artifact id from pbm for vmware-pbm for policy based management --- plugins/hypervisors/vmware/pom.xml | 2 +- vmware-base/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/hypervisors/vmware/pom.xml b/plugins/hypervisors/vmware/pom.xml index b0c1ca8cb79..c4dbb1d81b6 100644 --- a/plugins/hypervisors/vmware/pom.xml +++ b/plugins/hypervisors/vmware/pom.xml @@ -74,7 +74,7 @@ com.cloud.com.vmware - pbm + vmware-pbm ${cs.vmware.api.version} compile diff --git a/vmware-base/pom.xml b/vmware-base/pom.xml index 925a0031400..ef03d9ada22 100644 --- a/vmware-base/pom.xml +++ b/vmware-base/pom.xml @@ -77,7 +77,7 @@ com.cloud.com.vmware - pbm + vmware-pbm ${cs.vmware.api.version} compile From 0483e299e60aa503cd581be992cb53c63ef49b3e Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Thu, 6 Aug 2020 13:23:38 +0530 Subject: [PATCH 053/164] Deleted temporary files --- check_diff | 103 ----------------------------------------------------- 1 file changed, 103 deletions(-) delete mode 100644 check_diff diff --git a/check_diff b/check_diff deleted file mode 100644 index 78b27ecd755..00000000000 --- a/check_diff +++ /dev/null @@ -1,103 +0,0 @@ -diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java -index 47d3013f22..2dbf257fb2 100644 ---- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java -+++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java -@@ -5551,11 +5551,11 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir - throw new InvalidParameterValueException("Unable to find the VM by id=" + vmId); - } - -- if (vm.getState() != State.Stopped) { -- InvalidParameterValueException ex = new InvalidParameterValueException("VM is not Stopped, unable to migrate the vm having the specified id"); -- ex.addProxyObject(vm.getUuid(), "vmId"); -- throw ex; -- } -+// if (vm.getState() != State.Stopped) { -+// InvalidParameterValueException ex = new InvalidParameterValueException("VM is not Stopped, unable to migrate the vm having the specified id"); -+// ex.addProxyObject(vm.getUuid(), "vmId"); -+// throw ex; -+// } - - if (vm.getType() != VirtualMachine.Type.User) { - // OffLineVmwareMigration: *WHY* ? -@@ -5634,14 +5634,14 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir - throw new InvalidParameterValueException("Unable to find the VM by id=" + vmId); - } - // business logic -- if (vm.getState() != State.Running) { -- if (s_logger.isDebugEnabled()) { -- s_logger.debug("VM is not Running, unable to migrate the vm " + vm); -- } -- InvalidParameterValueException ex = new InvalidParameterValueException("VM is not Running, unable to migrate the vm with specified id"); -- ex.addProxyObject(vm.getUuid(), "vmId"); -- throw ex; -- } -+// if (vm.getState() != State.Running) { -+// if (s_logger.isDebugEnabled()) { -+// s_logger.debug("VM is not Running, unable to migrate the vm " + vm); -+// } -+// InvalidParameterValueException ex = new InvalidParameterValueException("VM is not Running, unable to migrate the vm with specified id"); -+// ex.addProxyObject(vm.getUuid(), "vmId"); -+// throw ex; -+// } - - checkIfHostOfVMIsInPrepareForMaintenanceState(vm.getHostId(), vmId, "Migrate"); - -@@ -5987,15 +5987,15 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir - } - - // OfflineVmwareMigration: this would be it ;) if multiple paths exist: unify -- if (vm.getState() != State.Running) { -- // OfflineVmwareMigration: and not vmware -- if (s_logger.isDebugEnabled()) { -- s_logger.debug("VM is not Running, unable to migrate the vm " + vm); -- } -- CloudRuntimeException ex = new CloudRuntimeException("VM is not Running, unable to migrate the vm with" + " specified id"); -- ex.addProxyObject(vm.getUuid(), "vmId"); -- throw ex; -- } -+// if (vm.getState() != State.Running) { -+// // OfflineVmwareMigration: and not vmware -+// if (s_logger.isDebugEnabled()) { -+// s_logger.debug("VM is not Running, unable to migrate the vm " + vm); -+// } -+// CloudRuntimeException ex = new CloudRuntimeException("VM is not Running, unable to migrate the vm with" + " specified id"); -+// ex.addProxyObject(vm.getUuid(), "vmId"); -+// throw ex; -+// } - - if(serviceOfferingDetailsDao.findDetail(vm.getServiceOfferingId(), GPU.Keys.pciDevice.toString()) != null) { - throw new InvalidParameterValueException("Live Migration of GPU enabled VM is not supported"); -diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java -index 62ecc9a5d5..36e102cb20 100644 ---- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java -+++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java -@@ -34,8 +34,6 @@ import java.util.concurrent.ExecutorService; - import java.util.concurrent.Executors; - import java.util.concurrent.Future; - --import com.vmware.vim25.VStorageObject; --import com.vmware.vim25.VStorageObjectConfigInfo; - import org.apache.commons.collections.CollectionUtils; - import org.apache.log4j.Logger; - import org.apache.commons.lang.StringUtils; -@@ -2430,13 +2428,13 @@ public class VirtualMachineMO extends BaseMO { - String deviceNumbering = getDeviceBusName(devices, device); - - s_logger.info("Disk backing : " + diskBackingInfo.getFileName() + " matches ==> " + deviceNumbering); -- if (((VirtualDisk) device).getVDiskId() == null) { -- s_logger.debug("vDiskid does not exist for volume " + vmdkDatastorePath + " registering the disk now"); -- VirtualStorageObjectManagerMO vStorageObjectManagerMO = new VirtualStorageObjectManagerMO(getOwnerDatacenter().first().getContext()); -- VStorageObject vStorageObject = vStorageObjectManagerMO.registerVirtualDisk(dsBackingFile, null, getOwnerDatacenter().first().getName()); -- VStorageObjectConfigInfo diskConfigInfo = vStorageObject.getConfig(); -- ((VirtualDisk) device).setVDiskId(diskConfigInfo.getId()); -- } -+// if (((VirtualDisk) device).getVDiskId() == null) { -+// s_logger.debug("vDiskid does not exist for volume " + vmdkDatastorePath + " registering the disk now"); -+// VirtualStorageObjectManagerMO vStorageObjectManagerMO = new VirtualStorageObjectManagerMO(getOwnerDatacenter().first().getContext()); -+// VStorageObject vStorageObject = vStorageObjectManagerMO.registerVirtualDisk(dsBackingFile, null, getOwnerDatacenter().first().getName()); -+// VStorageObjectConfigInfo diskConfigInfo = vStorageObject.getConfig(); -+// ((VirtualDisk) device).setVDiskId(diskConfigInfo.getId()); -+// } - return new Pair<>((VirtualDisk)device, deviceNumbering); - } - From 295ea5028214de1cbcaa9320b985a14450492e60 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Thu, 6 Aug 2020 14:12:17 +0530 Subject: [PATCH 054/164] Fixed unexpected token Exception --- .../src/main/java/com/cloud/dc/VsphereStoragePolicyVO.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/schema/src/main/java/com/cloud/dc/VsphereStoragePolicyVO.java b/engine/schema/src/main/java/com/cloud/dc/VsphereStoragePolicyVO.java index 1415d15ec77..5324de63bb3 100644 --- a/engine/schema/src/main/java/com/cloud/dc/VsphereStoragePolicyVO.java +++ b/engine/schema/src/main/java/com/cloud/dc/VsphereStoragePolicyVO.java @@ -14,7 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -package com.cloud.dc;; +package com.cloud.dc; import com.cloud.utils.DateUtil; import com.cloud.utils.db.GenericDao; From 2dd6e184d5e23ca1f55129030dd3200d10b7c6ee Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Thu, 6 Aug 2020 17:22:06 +0530 Subject: [PATCH 055/164] Fix simulator code for SimulatorStorageProcessor.java --- .../java/com/cloud/resource/SimulatorStorageProcessor.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/plugins/hypervisors/simulator/src/main/java/com/cloud/resource/SimulatorStorageProcessor.java b/plugins/hypervisors/simulator/src/main/java/com/cloud/resource/SimulatorStorageProcessor.java index e4ef4dfc1f6..16579bc3886 100644 --- a/plugins/hypervisors/simulator/src/main/java/com/cloud/resource/SimulatorStorageProcessor.java +++ b/plugins/hypervisors/simulator/src/main/java/com/cloud/resource/SimulatorStorageProcessor.java @@ -31,6 +31,7 @@ import org.apache.cloudstack.storage.command.CopyCmdAnswer; import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.command.CreateObjectAnswer; import org.apache.cloudstack.storage.command.CreateObjectCommand; +import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplainceCommand; import org.apache.cloudstack.storage.command.DeleteCommand; import org.apache.cloudstack.storage.command.DettachAnswer; import org.apache.cloudstack.storage.command.DettachCommand; @@ -269,4 +270,9 @@ public class SimulatorStorageProcessor implements StorageProcessor { public Answer copyVolumeFromPrimaryToPrimary(CopyCommand cmd) { return null; } + + @Override + public Answer CheckDataStoreStoragePolicyComplaince(CheckDataStoreStoragePolicyComplainceCommand cmd) { + return new Answer(cmd, true, null); + } } From ab171adfdd02c31ebff79199903bb54dd907bfd0 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Thu, 6 Aug 2020 23:56:11 +0530 Subject: [PATCH 056/164] Handle failure case of putting datastore cluster on maintenance mode --- .../java/com/cloud/storage/StorageManagerImpl.java | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index eedcf71983b..597ec58ef8f 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -1562,9 +1562,8 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C } }); List maintenanceSuccessfulStoragePools = new ArrayList<>(); - for (StoragePoolVO childDatastore : childDatastores) { - //FR41 need to handle when one of the primary stores is unable to put in maintenance mode - DataStore childStore = _dataStoreMgr.getDataStore(childDatastore.getId(), DataStoreRole.Primary); + for (Iterator iteratorChildDatastore = childDatastores.listIterator(); iteratorChildDatastore.hasNext(); ) { + DataStore childStore = _dataStoreMgr.getDataStore(iteratorChildDatastore.next().getId(), DataStoreRole.Primary); try { lifeCycle.maintain(childStore); } catch (Exception e) { @@ -1572,9 +1571,17 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C s_logger.debug(String.format("Exception on maintenance preparation of one of the child datastores in datastore cluster %d with error %s", primaryStorageId, e)); s_logger.debug(String.format("Cancelling the maintenance mode of child datastores in datastore cluster %d", primaryStorageId)); } + // Cancel maintenance mode of already prepared child storage pools + maintenanceSuccessfulStoragePools.add(childStore); for (DataStore dataStore: maintenanceSuccessfulStoragePools) { lifeCycle.cancelMaintain(dataStore); } + // Set back to Up state of remaining child storage pools + while (iteratorChildDatastore.hasNext()) { + StoragePoolVO childDatastore = iteratorChildDatastore.next(); + childDatastore.setStatus(StoragePoolStatus.Up); + _storagePoolDao.update(childDatastore.getId(), childDatastore); + } throw new CloudRuntimeException(String.format("Failed to prepare maintenance mode for datastore cluster %d with error %s %s", primaryStorageId, e.getMessage(), e)); } maintenanceSuccessfulStoragePools.add(childStore); From 201ebe88681a30837d3a19cb3212649d5a54630d Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Sun, 9 Aug 2020 13:43:48 +0530 Subject: [PATCH 057/164] Simulator failures fixing --- .../agent/api/ModifyStoragePoolAnswer.java | 3 +- .../provider/DefaultHostListener.java | 72 ++++++++++--------- 2 files changed, 39 insertions(+), 36 deletions(-) diff --git a/core/src/main/java/com/cloud/agent/api/ModifyStoragePoolAnswer.java b/core/src/main/java/com/cloud/agent/api/ModifyStoragePoolAnswer.java index f7a7e1ced06..be84cce152d 100644 --- a/core/src/main/java/com/cloud/agent/api/ModifyStoragePoolAnswer.java +++ b/core/src/main/java/com/cloud/agent/api/ModifyStoragePoolAnswer.java @@ -19,6 +19,7 @@ package com.cloud.agent.api; +import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -29,7 +30,7 @@ public class ModifyStoragePoolAnswer extends Answer { private Map templateInfo; private String localDatastoreName; private String poolType; - private List datastoreClusterChildren; + private List datastoreClusterChildren = new ArrayList<>();; public ModifyStoragePoolAnswer(ModifyStoragePoolCommand cmd, long capacityBytes, long availableBytes, Map tInfo) { super(cmd); diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java index 5c55a36c5b5..cb4ff74eed6 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java @@ -103,42 +103,44 @@ public class DefaultHostListener implements HypervisorHostListener { StoragePoolVO poolVO = this.primaryStoreDao.findById(poolId); updateStoragePoolHostVOAndDetails(poolVO, hostId, mspAnswer); - for (ModifyStoragePoolAnswer childDataStoreAnswer : ((ModifyStoragePoolAnswer) answer).getDatastoreClusterChildren()) { - StoragePoolInfo childStoragePoolInfo = childDataStoreAnswer.getPoolInfo(); - StoragePoolVO dataStoreVO = primaryStoreDao.findPoolByUUID(childStoragePoolInfo.getUuid()); - if (dataStoreVO != null) { - continue; + if (pool.getPoolType() == Storage.StoragePoolType.DatastoreCluster) { + for (ModifyStoragePoolAnswer childDataStoreAnswer : ((ModifyStoragePoolAnswer) answer).getDatastoreClusterChildren()) { + StoragePoolInfo childStoragePoolInfo = childDataStoreAnswer.getPoolInfo(); + StoragePoolVO dataStoreVO = primaryStoreDao.findPoolByUUID(childStoragePoolInfo.getUuid()); + if (dataStoreVO != null) { + continue; + } + dataStoreVO = new StoragePoolVO(); + dataStoreVO.setStorageProviderName(poolVO.getStorageProviderName()); + dataStoreVO.setHostAddress(childStoragePoolInfo.getHost()); + dataStoreVO.setPoolType(Storage.StoragePoolType.PreSetup); + dataStoreVO.setPath(childStoragePoolInfo.getHostPath()); + dataStoreVO.setPort(poolVO.getPort()); + dataStoreVO.setName(childStoragePoolInfo.getName()); + dataStoreVO.setUuid(childStoragePoolInfo.getUuid()); + dataStoreVO.setDataCenterId(poolVO.getDataCenterId()); + dataStoreVO.setPodId(poolVO.getPodId()); + dataStoreVO.setClusterId(poolVO.getClusterId()); + dataStoreVO.setStatus(StoragePoolStatus.Up); + dataStoreVO.setUserInfo(poolVO.getUserInfo()); + dataStoreVO.setManaged(poolVO.isManaged()); + dataStoreVO.setCapacityIops(poolVO.getCapacityIops()); + dataStoreVO.setCapacityBytes(childDataStoreAnswer.getPoolInfo().getCapacityBytes()); + dataStoreVO.setUsedBytes(childDataStoreAnswer.getPoolInfo().getCapacityBytes() - childDataStoreAnswer.getPoolInfo().getAvailableBytes()); + dataStoreVO.setHypervisor(poolVO.getHypervisor()); + dataStoreVO.setScope(poolVO.getScope()); + dataStoreVO.setParent(poolVO.getId()); + + Map details = new HashMap<>(); + if(StringUtils.isNotEmpty(childDataStoreAnswer.getPoolType())) { + details.put("pool_type", childDataStoreAnswer.getPoolType()); + } + + List storageTags = storagePoolTagsDao.getStoragePoolTags(poolId); + primaryStoreDao.persist(dataStoreVO, details, storageTags); + + updateStoragePoolHostVOAndDetails(dataStoreVO, hostId, childDataStoreAnswer); } - dataStoreVO = new StoragePoolVO(); - dataStoreVO.setStorageProviderName(poolVO.getStorageProviderName()); - dataStoreVO.setHostAddress(childStoragePoolInfo.getHost()); - dataStoreVO.setPoolType(Storage.StoragePoolType.PreSetup); - dataStoreVO.setPath(childStoragePoolInfo.getHostPath()); - dataStoreVO.setPort(poolVO.getPort()); - dataStoreVO.setName(childStoragePoolInfo.getName()); - dataStoreVO.setUuid(childStoragePoolInfo.getUuid()); - dataStoreVO.setDataCenterId(poolVO.getDataCenterId()); - dataStoreVO.setPodId(poolVO.getPodId()); - dataStoreVO.setClusterId(poolVO.getClusterId()); - dataStoreVO.setStatus(StoragePoolStatus.Up); - dataStoreVO.setUserInfo(poolVO.getUserInfo()); - dataStoreVO.setManaged(poolVO.isManaged()); - dataStoreVO.setCapacityIops(poolVO.getCapacityIops()); - dataStoreVO.setCapacityBytes(childDataStoreAnswer.getPoolInfo().getCapacityBytes()); - dataStoreVO.setUsedBytes(childDataStoreAnswer.getPoolInfo().getCapacityBytes() - childDataStoreAnswer.getPoolInfo().getAvailableBytes()); - dataStoreVO.setHypervisor(poolVO.getHypervisor()); - dataStoreVO.setScope(poolVO.getScope()); - dataStoreVO.setParent(poolVO.getId()); - - Map details = new HashMap<>(); - if(StringUtils.isNotEmpty(childDataStoreAnswer.getPoolType())) { - details.put("pool_type", childDataStoreAnswer.getPoolType()); - } - - List storageTags = storagePoolTagsDao.getStoragePoolTags(poolId); - primaryStoreDao.persist(dataStoreVO, details, storageTags); - - updateStoragePoolHostVOAndDetails(dataStoreVO, hostId, childDataStoreAnswer); } s_logger.info("Connection established between storage pool " + pool + " and host " + hostId); From b88df5ee5b6c5aafdfaa581268be64300fa3e39c Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Tue, 11 Aug 2020 15:35:07 +0530 Subject: [PATCH 058/164] Fixed a smoke test while taking snapshot --- .../java/com/cloud/storage/resource/VmwareStorageProcessor.java | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java index cd6edbe964b..0a9de9f5a78 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -1741,6 +1741,7 @@ public class VmwareStorageProcessor implements StorageProcessor { s_logger.error(msg); throw new Exception(msg); } + vmMo = clonedVm; } vmMo.exportVm(exportPath, exportName, false, false); From f320cf3c597a3c88a231c7c249557899e08e5a60 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Wed, 12 Aug 2020 00:34:58 +0530 Subject: [PATCH 059/164] Remove registering virtual disk code --- .../vmware/mo/VirtualMachineMO.java | 46 ------------------- 1 file changed, 46 deletions(-) diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java index c4e1493a5f4..4650be3d73d 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java @@ -61,8 +61,6 @@ import com.vmware.vim25.ParaVirtualSCSIController; import com.vmware.vim25.PropertyFilterSpec; import com.vmware.vim25.PropertySpec; import com.vmware.vim25.TraversalSpec; -import com.vmware.vim25.VStorageObject; -import com.vmware.vim25.VStorageObjectConfigInfo; import com.vmware.vim25.VirtualBusLogicController; import com.vmware.vim25.VirtualCdrom; import com.vmware.vim25.VirtualCdromIsoBackingInfo; @@ -2474,7 +2472,6 @@ public class VirtualMachineMO extends BaseMO { String deviceNumbering = getDeviceBusName(devices, device); s_logger.info("Disk backing : " + diskBackingInfo.getFileName() + " matches ==> " + deviceNumbering); - registerVirtualDisk((VirtualDisk) device, dsBackingFile); return new Pair<>((VirtualDisk)device, deviceNumbering); } @@ -2547,7 +2544,6 @@ public class VirtualMachineMO extends BaseMO { String deviceNumbering = getDeviceBusName(devices, device); s_logger.info("Disk backing : " + diskBackingInfo.getFileName() + " matches ==> " + deviceNumbering); - registerVirtualDisk((VirtualDisk) device, dsBackingFile); return new Pair((VirtualDisk)device, deviceNumbering); } } else { @@ -2555,7 +2551,6 @@ public class VirtualMachineMO extends BaseMO { String deviceNumbering = getDeviceBusName(devices, device); s_logger.info("Disk backing : " + diskBackingInfo.getFileName() + " matches ==> " + deviceNumbering); - registerVirtualDisk((VirtualDisk) device, dsBackingFile); return new Pair((VirtualDisk)device, deviceNumbering); } } @@ -2570,20 +2565,6 @@ public class VirtualMachineMO extends BaseMO { return null; } - public void registerVirtualDisk(VirtualDisk device, DatastoreFile dsBackingFile) { - if (((VirtualDisk) device).getVDiskId() == null) { - try { - s_logger.debug("vDiskid does not exist for volume " + dsBackingFile.getFileName() + " registering the disk now"); - VirtualStorageObjectManagerMO vStorageObjectManagerMO = new VirtualStorageObjectManagerMO(getOwnerDatacenter().first().getContext()); - VStorageObject vStorageObject = vStorageObjectManagerMO.registerVirtualDisk(dsBackingFile, null, getOwnerDatacenter().first().getName()); - VStorageObjectConfigInfo diskConfigInfo = vStorageObject.getConfig(); - ((VirtualDisk) device).setVDiskId(diskConfigInfo.getId()); - } catch (Exception e) { - s_logger.warn("Exception while trying to register a disk as first class disk to get the unique identifier, main operation still continues: " + e.getMessage()); - } - } - } - public String getDiskCurrentTopBackingFileInChain(String deviceBusName) throws Exception { List devices = _context.getVimClient().getDynamicProperty(_mor, "config.hardware.device"); if (devices != null && devices.size() > 0) { @@ -2633,14 +2614,11 @@ public class VirtualMachineMO extends BaseMO { VirtualDeviceBackingInfo backingInfo = ((VirtualDisk)device).getBacking(); if (backingInfo instanceof VirtualDiskFlatVer2BackingInfo) { VirtualDiskFlatVer2BackingInfo diskBackingInfo = (VirtualDiskFlatVer2BackingInfo)backingInfo; - String diskBackingFileName = diskBackingInfo.getFileName(); while (diskBackingInfo != null) { String deviceBusName = getDeviceBusName(devices, device); builder.addDisk(deviceBusName, diskBackingInfo.getFileName()); diskBackingInfo = diskBackingInfo.getParent(); } - DatastoreFile dsBackingFile = new DatastoreFile(diskBackingFileName); - registerVirtualDisk((VirtualDisk) device, dsBackingFile); } } } @@ -2659,8 +2637,6 @@ public class VirtualMachineMO extends BaseMO { VirtualDeviceBackingInfo backingInfo = ((VirtualDisk)device).getBacking(); if (backingInfo instanceof VirtualDiskFlatVer2BackingInfo) { VirtualDiskFlatVer2BackingInfo diskBackingInfo = (VirtualDiskFlatVer2BackingInfo)backingInfo; - DatastoreFile dsBackingFile = new DatastoreFile(diskBackingInfo.getFileName()); - registerVirtualDisk((VirtualDisk) device, dsBackingFile); disks.add(new Pair(new Integer(device.getKey()), diskBackingInfo.getDatastore())); } } @@ -2769,10 +2745,6 @@ public class VirtualMachineMO extends BaseMO { for (VirtualDevice device : devices) { if (device instanceof VirtualDisk) { - VirtualDeviceBackingInfo backingInfo = device.getBacking(); - VirtualDiskFlatVer2BackingInfo diskBackingInfo = (VirtualDiskFlatVer2BackingInfo)backingInfo; - DatastoreFile dsBackingFile = new DatastoreFile(diskBackingInfo.getFileName()); - registerVirtualDisk((VirtualDisk) device, dsBackingFile); virtualDisks.add((VirtualDisk)device); } } @@ -2807,7 +2779,6 @@ public class VirtualMachineMO extends BaseMO { reConfigSpec.getDeviceChange().add(deviceConfigSpec); } - registerVirtualDisk((VirtualDisk) device, dsBackingFile); } } @@ -2835,23 +2806,6 @@ public class VirtualMachineMO extends BaseMO { if (devices != null && devices.size() > 0) { for (VirtualDevice device : devices) { if (device instanceof VirtualDisk) { - if (((VirtualDisk) device).getVDiskId() == null) { - try { - // Register as first class disk - VirtualDeviceBackingInfo backingInfo = device.getBacking(); - if (backingInfo instanceof VirtualDiskFlatVer2BackingInfo) { - VirtualDiskFlatVer2BackingInfo diskBackingInfo = (VirtualDiskFlatVer2BackingInfo) backingInfo; - DatastoreFile dsBackingFile = new DatastoreFile(diskBackingInfo.getFileName()); - s_logger.debug("vDiskid does not exist for volume " + diskBackingInfo.getFileName() + " registering the disk now"); - VirtualStorageObjectManagerMO vStorageObjectManagerMO = new VirtualStorageObjectManagerMO(getOwnerDatacenter().first().getContext()); - VStorageObject vStorageObject = vStorageObjectManagerMO.registerVirtualDisk(dsBackingFile, null, getOwnerDatacenter().first().getName()); - VStorageObjectConfigInfo diskConfigInfo = vStorageObject.getConfig(); - ((VirtualDisk) device).setVDiskId(diskConfigInfo.getId()); - } - } catch (Exception e) { - s_logger.warn("Exception while trying to register a disk as first class disk to get the unique identifier, main operation still continues: " + e.getMessage()); - } - } deviceList.add((VirtualDisk)device); } } From af84743fd956b1dc63192d37196b35cc1e3ae95a Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Thu, 13 Aug 2020 22:28:16 +0530 Subject: [PATCH 060/164] Fix addition of datastore clusters not to configure it as non NFS storage --- .../com/cloud/hypervisor/vmware/resource/VmwareResource.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 977f8bab151..bab45bf492b 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -4929,7 +4929,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa ManagedObjectReference morDatastore = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, pool.getUuid()); if (morDatastore == null) { - morDatastore = hyperHost.mountDatastore((pool.getType() == StoragePoolType.VMFS || pool.getType() == StoragePoolType.PreSetup), pool.getHost(), pool.getPort(), pool.getPath(), pool.getUuid().replace("-", ""), true); + morDatastore = hyperHost.mountDatastore((pool.getType() == StoragePoolType.VMFS || pool.getType() == StoragePoolType.PreSetup || pool.getType() == StoragePoolType.DatastoreCluster), pool.getHost(), pool.getPort(), pool.getPath(), pool.getUuid().replace("-", ""), true); } assert (morDatastore != null); From d4d372a9a4910a3afb527634963aed1c1a6d1e6d Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Sun, 16 Aug 2020 00:05:55 +0530 Subject: [PATCH 061/164] Fix addition of datastores with invalid vCenter server details --- plugins/storage/volume/default/pom.xml | 6 +++++ ...oudStackPrimaryDataStoreLifeCycleImpl.java | 25 +++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/plugins/storage/volume/default/pom.xml b/plugins/storage/volume/default/pom.xml index 51450e0f8a8..c620e72ad35 100644 --- a/plugins/storage/volume/default/pom.xml +++ b/plugins/storage/volume/default/pom.xml @@ -33,6 +33,12 @@ cloud-engine-storage-volume ${project.version} + + org.apache.cloudstack + cloud-plugin-hypervisor-vmware + 4.15.0.0-SNAPSHOT + compile + diff --git a/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java b/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java index 955af589c19..4195b4a72a3 100644 --- a/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java +++ b/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java @@ -30,6 +30,10 @@ import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.hypervisor.vmware.VmwareDatacenterVO; +import com.cloud.hypervisor.vmware.VmwareDatacenterZoneMapVO; +import com.cloud.hypervisor.vmware.dao.VmwareDatacenterDao; +import com.cloud.hypervisor.vmware.dao.VmwareDatacenterZoneMapDao; import com.cloud.resource.ResourceManager; import com.cloud.server.ManagementServer; import com.cloud.storage.OCFS2Manager; @@ -122,6 +126,10 @@ public class CloudStackPrimaryDataStoreLifeCycleImpl implements PrimaryDataStore StoragePoolAutomation storagePoolAutmation; @Inject protected HostDao _hostDao; + @Inject + private VmwareDatacenterZoneMapDao vmwareDatacenterZoneMapDao; + @Inject + private VmwareDatacenterDao vmwareDcDao; @SuppressWarnings("unchecked") @Override @@ -249,11 +257,17 @@ public class CloudStackPrimaryDataStoreLifeCycleImpl implements PrimaryDataStore parameters.setPath(hostPath.replaceFirst("/", "")); parameters.setUserInfo(userInfo); } else if (scheme.equalsIgnoreCase("PreSetup")) { + if (StringUtils.isNotBlank(hypervisorType) && HypervisorType.getType(hypervisorType).equals(HypervisorType.VMware)) { + validateVcenterDetails(zoneId, storageHost); + } parameters.setType(StoragePoolType.PreSetup); parameters.setHost(storageHost); parameters.setPort(0); parameters.setPath(hostPath); } else if (scheme.equalsIgnoreCase("DatastoreCluster")) { + if (StringUtils.isNotBlank(hypervisorType) && HypervisorType.getType(hypervisorType).equals(HypervisorType.VMware)) { + validateVcenterDetails(zoneId, storageHost); + } parameters.setType(StoragePoolType.DatastoreCluster); parameters.setHost(storageHost); parameters.setPort(0); @@ -357,6 +371,17 @@ public class CloudStackPrimaryDataStoreLifeCycleImpl implements PrimaryDataStore return dataStoreHelper.createPrimaryDataStore(parameters); } + private void validateVcenterDetails(long zoneId, String storageHost) { + VmwareDatacenterZoneMapVO vmwareDcZoneMap = vmwareDatacenterZoneMapDao.findByZoneId(zoneId); + if (vmwareDcZoneMap != null) { + Long associatedVmwareDcId = vmwareDcZoneMap.getVmwareDcId(); + VmwareDatacenterVO associatedVmwareDc = vmwareDcDao.findById(associatedVmwareDcId); + if (!associatedVmwareDc.getVcenterHost().equals(storageHost)) { + throw new InvalidParameterValueException("Provided vCenter server details does not match with the existing vCenter in zone id: " + zoneId); + } + } + } + protected boolean createStoragePool(long hostId, StoragePool pool) { s_logger.debug("creating pool " + pool.getName() + " on host " + hostId); From d48cab75ab44f34eecb1dcffda82e5570bb41288 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Sun, 16 Aug 2020 23:33:57 +0530 Subject: [PATCH 062/164] Fix findstoragepoolsformigration cmd not to list the parent datastore cluster as suitable storagepool --- .../com/cloud/server/ManagementServerImpl.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index f2eaeaf3f33..83f3fc05d98 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -1488,21 +1488,26 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe } else { suitablePools = allPools; } - abstractDataStoreClustersList((List) allPools); - abstractDataStoreClustersList((List) suitablePools); + List avoidPools = new ArrayList<>(); + if (srcVolumePool.getParent() != 0L) { + StoragePool datastoreCluster = _poolDao.findById(srcVolumePool.getParent()); + avoidPools.add(datastoreCluster); + } + abstractDataStoreClustersList((List) allPools, new ArrayList()); + abstractDataStoreClustersList((List) suitablePools, avoidPools); return new Pair, List>(allPools, suitablePools); } - private void abstractDataStoreClustersList(List storagePools) { + private void abstractDataStoreClustersList(List storagePools, List avoidPools) { Predicate childDatastorePredicate = pool -> (pool.getParent() != 0); List childDatastores = storagePools.stream().filter(childDatastorePredicate).collect(Collectors.toList()); - + storagePools.removeAll(avoidPools); if (!childDatastores.isEmpty()) { storagePools.removeAll(childDatastores); Set parentStoragePoolIds = childDatastores.stream().map(mo -> mo.getParent()).collect(Collectors.toSet()); for (Long parentStoragePoolId : parentStoragePoolIds) { StoragePool parentPool = _poolDao.findById(parentStoragePoolId); - if (!storagePools.contains(parentPool)) + if (!storagePools.contains(parentPool) && !avoidPools.contains(parentPool)) storagePools.add(parentPool); } } From 0a412ac6dad2541fde53dfb68208678e480d07ca Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Mon, 17 Aug 2020 01:49:09 +0530 Subject: [PATCH 063/164] Handle volume copy from secondary to primary in case of vSAN --- .../cloud/storage/resource/VmwareStorageProcessor.java | 2 +- .../java/com/cloud/hypervisor/vmware/mo/DatastoreMO.java | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java index 0a9de9f5a78..68f63619cab 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -3483,7 +3483,7 @@ public class VmwareStorageProcessor implements StorageProcessor { throw new Exception("Unable to create container VM for volume creation"); } - clonedVm.moveAllVmDiskFiles(primaryDsMo, "", false); + clonedVm.moveAllVmDiskFiles(primaryDsMo, HypervisorHostHelper.VSPHERE_DATASTORE_BASE_FOLDER, false); clonedVm.detachAllDisks(); return _storage.getSize(srcOVFFileName); } finally { diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatastoreMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatastoreMO.java index b5955a6d709..96ed988c35d 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatastoreMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatastoreMO.java @@ -260,8 +260,13 @@ public class DatastoreMO extends BaseMO { destFullPath = String.format("[%s] %s", destDsName, destFilePath); DatastoreMO srcDsMo = new DatastoreMO(_context, morDestDs); - if (!srcDsMo.fileExists(srcFullPath)) { - s_logger.error(String.format("Cannot move file to destination datastore due to file %s does not exists", srcFullPath)); + try { + if (!srcDsMo.fileExists(srcFullPath)) { + s_logger.error(String.format("Cannot move file to destination datastore due to file %s does not exists", srcFullPath)); + return false; + } + } catch (Exception e) { + s_logger.error(String.format("Cannot move file to destination datastore due to file %s due to exeception %s", srcFullPath, e.getMessage())); return false; } From 85d440d9738cd133230243558a7859e8306a926d Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Tue, 18 Aug 2020 02:25:45 +0530 Subject: [PATCH 064/164] Fix list volume response to list datastore cluster name and id instead of child datastore --- .../com/cloud/api/query/dao/VolumeJoinDaoImpl.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDaoImpl.java index c7977356379..2ee0ca1b1f4 100644 --- a/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/VolumeJoinDaoImpl.java @@ -24,6 +24,8 @@ import javax.inject.Inject; import org.apache.cloudstack.api.ResponseObject.ResponseView; import org.apache.cloudstack.api.response.VolumeResponse; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -51,6 +53,8 @@ public class VolumeJoinDaoImpl extends GenericDaoBaseWithTagInformation volSearch; @@ -220,6 +224,14 @@ public class VolumeJoinDaoImpl extends GenericDaoBaseWithTagInformation Date: Tue, 18 Aug 2020 11:24:39 +0530 Subject: [PATCH 065/164] Removed unused imports caused due to some merge conflicts --- server/src/main/java/com/cloud/api/query/QueryManagerImpl.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java index 0fd7aa1398d..06124f3755d 100644 --- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java @@ -31,8 +31,6 @@ import java.util.stream.Stream; import javax.inject.Inject; -import com.cloud.projects.dao.ProjectInvitationDao; -import com.cloud.user.dao.UserDao; import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.affinity.AffinityGroupDomainMapVO; import org.apache.cloudstack.affinity.AffinityGroupResponse; From 570f3214b83c6210edc4ba01a6306c4757138f92 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Tue, 18 Aug 2020 22:37:27 +0530 Subject: [PATCH 066/164] Handle VMFS6 sesparse format disk files --- .../com/cloud/storage/resource/VmwareStorageProcessor.java | 5 +++-- plugins/storage/volume/default/pom.xml | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java index 68f63619cab..2c4d9a14381 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -851,8 +851,9 @@ public class VmwareStorageProcessor implements StorageProcessor { String[] vmwareLayoutFilePair = VmwareStorageLayoutHelper.getVmdkFilePairDatastorePath(dsMo, vmdkName, vmdkFileBaseName, VmwareStorageLayoutType.VMWARE, !_fullCloneFlag); String[] legacyCloudStackLayoutFilePair = VmwareStorageLayoutHelper.getVmdkFilePairDatastorePath(dsMo, vmdkName, vmdkFileBaseName, VmwareStorageLayoutType.CLOUDSTACK_LEGACY, !_fullCloneFlag); - dsMo.moveDatastoreFile(vmwareLayoutFilePair[0], dcMo.getMor(), dsMo.getMor(), legacyCloudStackLayoutFilePair[0], dcMo.getMor(), true); - dsMo.moveDatastoreFile(vmwareLayoutFilePair[1], dcMo.getMor(), dsMo.getMor(), legacyCloudStackLayoutFilePair[1], dcMo.getMor(), true); + for (int i=0; i org.apache.cloudstack cloud-plugin-hypervisor-vmware - 4.15.0.0-SNAPSHOT + ${project.version} compile From ebe0e2bb24f02d5b3e547a6759a667965adbf5b9 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Wed, 19 Aug 2020 23:57:01 +0530 Subject: [PATCH 067/164] Keep retry logic for make directory method which is returning false positive yet times --- .../com/cloud/hypervisor/vmware/mo/DatastoreMO.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatastoreMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatastoreMO.java index 96ed988c35d..fb9e83cf308 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatastoreMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatastoreMO.java @@ -136,6 +136,16 @@ public class DatastoreMO extends BaseMO { fullPath = String.format("[%s] %s", datastoreName, path); _context.getService().makeDirectory(morFileManager, fullPath, morDc, true); + + int retry = 2; + for (int i = 0; i < retry; i++) { + DatastoreFile datastoreFile = new DatastoreFile(fullPath); + if (!folderExists(String.format("[%s]", datastoreName), datastoreFile.getFileName())) { + _context.getService().makeDirectory(morFileManager, fullPath, morDc, true); + } else { + return; + } + } } String getDatastoreRootPath() throws Exception { From d2d9b3c72f625f1efcbe6f20032e6002da3004ae Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Thu, 20 Aug 2020 00:14:13 +0530 Subject: [PATCH 068/164] Fix proper error message on revert volume snapshot operation in case of any hypervisor other than KVM --- .../java/com/cloud/storage/snapshot/SnapshotManagerImpl.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java index 35ec665b97d..1eb72e2a24c 100755 --- a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java @@ -311,7 +311,8 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement if (snapshotStrategy == null) { s_logger.error("Unable to find snaphot strategy to handle snapshot with id '" + snapshotId + "'"); - return null; + String errorMsg = String.format("Revert snapshot command failed for snapshot with id %d, because this command is supported only for KVM hypervisor", snapshotId); + throw new CloudRuntimeException(errorMsg); } boolean result = snapshotStrategy.revertSnapshot(snapshotInfo); From c16b40e33712777ebcc2e32050ec4090956fe979 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Thu, 20 Aug 2020 04:48:54 +0530 Subject: [PATCH 069/164] Fix list volumes cmd and update volume cmd to accept datastore cluster as storgeid --- .../com/cloud/api/query/QueryManagerImpl.java | 18 ++++++++++++++++-- .../cloud/storage/VolumeApiServiceImpl.java | 8 +++++++- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java index 06124f3755d..005968949c7 100644 --- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java @@ -1955,7 +1955,14 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q sb.and("instanceId", sb.entity().getVmId(), SearchCriteria.Op.EQ); sb.and("dataCenterId", sb.entity().getDataCenterId(), SearchCriteria.Op.EQ); sb.and("podId", sb.entity().getPodId(), SearchCriteria.Op.EQ); - sb.and("storageId", sb.entity().getPoolUuid(), SearchCriteria.Op.EQ); + if (storageId != null) { + StoragePoolVO poolVO = _storagePoolDao.findByUuid(storageId); + if (poolVO.getPoolType() == Storage.StoragePoolType.DatastoreCluster) { + sb.and("storageId", sb.entity().getPoolUuid(), SearchCriteria.Op.IN); + } else { + sb.and("storageId", sb.entity().getPoolUuid(), SearchCriteria.Op.EQ); + } + } sb.and("diskOfferingId", sb.entity().getDiskOfferingId(), SearchCriteria.Op.EQ); sb.and("display", sb.entity().isDisplayVolume(), SearchCriteria.Op.EQ); sb.and("state", sb.entity().getState(), SearchCriteria.Op.EQ); @@ -2024,7 +2031,14 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q } if (storageId != null) { - sc.setParameters("storageId", storageId); + StoragePoolVO poolVO = _storagePoolDao.findByUuid(storageId); + if (poolVO.getPoolType() == Storage.StoragePoolType.DatastoreCluster) { + List childDatastores = _storagePoolDao.listChildStoragePoolsInDatastoreCluster(poolVO.getId()); + List childDatastoreIds = childDatastores.stream().map(mo -> mo.getUuid()).collect(Collectors.toList()); + sc.setParameters("storageId", childDatastoreIds.toArray()); + } else { + sc.setParameters("storageId", storageId); + } } if (clusterId != null) { diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index e18c3553b01..4127b9f5fcc 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -1777,7 +1777,13 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic if (pool.getDataCenterId() != volume.getDataCenterId()) { throw new InvalidParameterValueException("Invalid storageId specified; refers to the pool outside of the volume's zone"); } - volume.setPoolId(pool.getId()); + if (pool.getPoolType() == Storage.StoragePoolType.DatastoreCluster) { + List childDatastores = _storagePoolDao.listChildStoragePoolsInDatastoreCluster(storageId); + Collections.shuffle(childDatastores); + volume.setPoolId(childDatastores.get(0).getId()); + } else { + volume.setPoolId(pool.getId()); + } } if (customId != null) { From 38b9618e4b9b6484bfbd01ac6714a64ce33eecfe Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Fri, 21 Aug 2020 01:02:48 +0530 Subject: [PATCH 070/164] Fix destination hostname for stopped VMs in case of reset ssh keys --- .../src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java | 3 ++- .../com/cloud/network/element/CloudZonesNetworkElement.java | 2 +- .../com/cloud/network/element/ConfigDriveNetworkElement.java | 2 +- .../main/java/com/cloud/network/router/CommandSetupHelper.java | 2 +- server/src/main/java/com/cloud/vm/UserVmManagerImpl.java | 2 +- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 380a06e6152..ffd5878bbf6 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -2955,7 +2955,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac final String serviceOffering = _serviceOfferingDao.findByIdIncludingRemoved(vm.getId(), vm.getServiceOfferingId()).getDisplayText(); boolean isWindows = _guestOSCategoryDao.findById(_guestOSDao.findById(vm.getGuestOSId()).getCategoryId()).getName().equalsIgnoreCase("Windows"); vmData = _networkModel.generateVmData(userVm.getUserData(), serviceOffering, vm.getDataCenterId(), vm.getInstanceName(), vm.getHostName(), vm.getId(), - vm.getUuid(), defaultNic.getMacAddress(), userVm.getDetail("SSH.PublicKey"), (String) profile.getParameter(VirtualMachineProfile.Param.VmPassword), isWindows, VirtualMachineManager.getHypervisorHostname(destination.getHost().getName())); + vm.getUuid(), defaultNic.getMacAddress(), userVm.getDetail("SSH.PublicKey"), (String) profile.getParameter(VirtualMachineProfile.Param.VmPassword), isWindows, + VirtualMachineManager.getHypervisorHostname(destination.getHost() != null ? destination.getHost().getName() : "")); String vmName = vm.getInstanceName(); String configDriveIsoRootFolder = "/tmp"; String isoFile = configDriveIsoRootFolder + "/" + vmName + "/configDrive/" + vmName + ".iso"; diff --git a/server/src/main/java/com/cloud/network/element/CloudZonesNetworkElement.java b/server/src/main/java/com/cloud/network/element/CloudZonesNetworkElement.java index 3ed5fcc7d43..848340b1a24 100644 --- a/server/src/main/java/com/cloud/network/element/CloudZonesNetworkElement.java +++ b/server/src/main/java/com/cloud/network/element/CloudZonesNetworkElement.java @@ -218,7 +218,7 @@ public class CloudZonesNetworkElement extends AdapterBase implements NetworkElem } String serviceOffering = _serviceOfferingDao.findByIdIncludingRemoved(uservm.getServiceOfferingId()).getDisplayText(); String zoneName = _dcDao.findById(network.getDataCenterId()).getName(); - String destHostname = VirtualMachineManager.getHypervisorHostname(dest.getHost().getName()); + String destHostname = VirtualMachineManager.getHypervisorHostname(dest.getHost() != null ? dest.getHost().getName() : ""); cmds.addCommand( "vmdata", generateVmDataCommand(nic.getIPv4Address(), userData, serviceOffering, zoneName, nic.getIPv4Address(), uservm.getHostName(), uservm.getInstanceName(), diff --git a/server/src/main/java/com/cloud/network/element/ConfigDriveNetworkElement.java b/server/src/main/java/com/cloud/network/element/ConfigDriveNetworkElement.java index 4daeda6a530..7482eca27a8 100644 --- a/server/src/main/java/com/cloud/network/element/ConfigDriveNetworkElement.java +++ b/server/src/main/java/com/cloud/network/element/ConfigDriveNetworkElement.java @@ -369,7 +369,7 @@ public class ConfigDriveNetworkElement extends AdapterBase implements NetworkEle if (userVm != null) { final boolean isWindows = isWindows(userVm.getGuestOSId()); List vmData = _networkModel.generateVmData(userVm.getUserData(), _serviceOfferingDao.findById(userVm.getServiceOfferingId()).getName(), userVm.getDataCenterId(), userVm.getInstanceName(), vm.getHostName(), vm.getId(), - vm.getUuid(), nic.getMacAddress(), userVm.getDetail("SSH.PublicKey"), (String) vm.getParameter(VirtualMachineProfile.Param.VmPassword), isWindows, VirtualMachineManager.getHypervisorHostname(dest.getHost().getName())); + vm.getUuid(), nic.getMacAddress(), userVm.getDetail("SSH.PublicKey"), (String) vm.getParameter(VirtualMachineProfile.Param.VmPassword), isWindows, VirtualMachineManager.getHypervisorHostname(dest.getHost() != null ? dest.getHost().getName() : "")); vm.setVmData(vmData); vm.setConfigDriveLabel(VirtualMachineManager.VmConfigDriveLabel.value()); createConfigDriveIso(vm, dest, diskToUse); diff --git a/server/src/main/java/com/cloud/network/router/CommandSetupHelper.java b/server/src/main/java/com/cloud/network/router/CommandSetupHelper.java index 00da0d3a102..4adf1406afb 100644 --- a/server/src/main/java/com/cloud/network/router/CommandSetupHelper.java +++ b/server/src/main/java/com/cloud/network/router/CommandSetupHelper.java @@ -195,7 +195,7 @@ public class CommandSetupHelper { final IPAddressVO staticNatIp = _ipAddressDao.findByVmIdAndNetworkId(nic.getNetworkId(), vm.getId()); Host host = _hostDao.findById(vm.getHostId()); - String destHostname = VirtualMachineManager.getHypervisorHostname(host.getName()); + String destHostname = VirtualMachineManager.getHypervisorHostname(host != null ? host.getName() : ""); cmds.addCommand( "vmdata", generateVmDataCommand(router, nic.getIPv4Address(), vm.getUserData(), serviceOffering, zoneName, diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 862190e3cb3..525931deddb 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -4356,7 +4356,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir if (_networkModel.isSharedNetworkWithoutServices(network.getId())) { final String serviceOffering = _serviceOfferingDao.findByIdIncludingRemoved(vm.getId(), vm.getServiceOfferingId()).getDisplayText(); boolean isWindows = _guestOSCategoryDao.findById(_guestOSDao.findById(vm.getGuestOSId()).getCategoryId()).getName().equalsIgnoreCase("Windows"); - String destHostname = VirtualMachineManager.getHypervisorHostname(dest.getHost().getName()); + String destHostname = VirtualMachineManager.getHypervisorHostname(dest.getHost() != null ? dest.getHost().getName() : ""); List vmData = _networkModel.generateVmData(vm.getUserData(), serviceOffering, vm.getDataCenterId(), vm.getInstanceName(), vm.getHostName(), vm.getId(), vm.getUuid(), defaultNic.getIPv4Address(), vm.getDetail(VmDetailConstants.SSH_PUBLIC_KEY), (String) profile.getParameter(VirtualMachineProfile.Param.VmPassword), isWindows, destHostname); String vmName = vm.getInstanceName(); From 19745ea0497e3f078151e0ba553c8fd9a83d069e Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Mon, 24 Aug 2020 16:19:57 +0530 Subject: [PATCH 071/164] Fix enable primary datastore maintenance command seriliaztion on it --- .../storage/allocator/AbstractStoragePoolAllocator.java | 9 +++++++++ .../main/java/com/cloud/storage/StorageManagerImpl.java | 9 +++++++++ .../com/cloud/storage/StoragePoolAutomationImpl.java | 4 ++-- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java index 114385bb3c2..cfe32c2bd07 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java @@ -27,6 +27,8 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; import com.cloud.exception.StorageUnavailableException; +import com.cloud.storage.StoragePoolStatus; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.log4j.Logger; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; @@ -222,6 +224,13 @@ public abstract class AbstractStoragePoolAllocator extends AdapterBase implement if (pool.getPoolType() == Storage.StoragePoolType.DatastoreCluster && storageMgr.isStoragePoolDatastoreClusterParent(pool)) { return false; } + // Skip the storage pool whose parent datastore cluster is not in UP state. + if (pool.getParent() != 0L) { + StoragePoolVO datastoreCluster = storagePoolDao.findById(pool.getParent()); + if (datastoreCluster == null || (datastoreCluster != null && datastoreCluster.getStatus() != StoragePoolStatus.Up)) { + return false; + } + } try { boolean isStoragePoolStoragepolicyComplaince = storageMgr.isStoragePoolComplaintWithStoragePolicy(requestVolumes, pool); diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index 597ec58ef8f..7b00e3491c5 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -1541,6 +1541,9 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C DataStore store = _dataStoreMgr.getDataStore(primaryStorage.getId(), DataStoreRole.Primary); if (primaryStorage.getPoolType() == StoragePoolType.DatastoreCluster) { + if (primaryStorage.getStatus() == StoragePoolStatus.PrepareForMaintenance) { + throw new CloudRuntimeException(String.format("There is already a job running for preparation for maintenance of the storage pool %s", primaryStorage.getUuid())); + } handlePrepareDatastoreCluserMaintenance(lifeCycle, primaryStorageId); } lifeCycle.maintain(store); @@ -1549,6 +1552,10 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C } private void handlePrepareDatastoreCluserMaintenance(DataStoreLifeCycle lifeCycle, Long primaryStorageId) { + StoragePoolVO datastoreCluster = _storagePoolDao.findById(primaryStorageId); + datastoreCluster.setStatus(StoragePoolStatus.PrepareForMaintenance); + _storagePoolDao.update(datastoreCluster.getId(), datastoreCluster); + // Before preparing the datastorecluster to maintenance mode, the storagepools in the datastore cluster needs to put in maintenance List childDatastores = _storagePoolDao.listChildStoragePoolsInDatastoreCluster(primaryStorageId); Transaction.execute(new TransactionCallbackNoReturn() { @@ -1611,6 +1618,8 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C DataStoreLifeCycle lifeCycle = provider.getDataStoreLifeCycle(); DataStore store = _dataStoreMgr.getDataStore(primaryStorage.getId(), DataStoreRole.Primary); if (primaryStorage.getPoolType() == StoragePoolType.DatastoreCluster) { + primaryStorage.setStatus(StoragePoolStatus.Up); + _storagePoolDao.update(primaryStorage.getId(), primaryStorage); //FR41 need to handle when one of the primary stores is unable to cancel the maintenance mode List childDatastores = _storagePoolDao.listChildStoragePoolsInDatastoreCluster(primaryStorageId); for (StoragePoolVO childDatastore : childDatastores) { diff --git a/server/src/main/java/com/cloud/storage/StoragePoolAutomationImpl.java b/server/src/main/java/com/cloud/storage/StoragePoolAutomationImpl.java index 15a72590fc4..3e8822e67eb 100644 --- a/server/src/main/java/com/cloud/storage/StoragePoolAutomationImpl.java +++ b/server/src/main/java/com/cloud/storage/StoragePoolAutomationImpl.java @@ -117,7 +117,7 @@ public class StoragePoolAutomationImpl implements StoragePoolAutomation { spes = primaryDataStoreDao.listBy(pool.getDataCenterId(), pool.getPodId(), pool.getClusterId(), ScopeType.CLUSTER); } for (StoragePoolVO sp : spes) { - if (sp.getParent() != pool.getParent()) { // If Datastore cluster is tried to prepare for maintenance then child storage pools are also kept in PrepareForMaintenance mode + if (sp.getParent() != pool.getParent() && sp.getId() != pool.getParent()) { // If Datastore cluster is tried to prepare for maintenance then child storage pools are also kept in PrepareForMaintenance mode if (sp.getStatus() == StoragePoolStatus.PrepareForMaintenance) { throw new CloudRuntimeException("Only one storage pool in a cluster can be in PrepareForMaintenance mode, " + sp.getId() + " is already in PrepareForMaintenance mode "); @@ -321,7 +321,7 @@ public class StoragePoolAutomationImpl implements StoragePoolAutomation { } } else { if (s_logger.isDebugEnabled()) { - s_logger.debug("ModifyStoragePool add secceeded"); + s_logger.debug("ModifyStoragePool add succeeded"); } } } From 40934ba9ff1b84e5dde559deabb4bc8e56193ce0 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Thu, 27 Aug 2020 12:53:06 +0530 Subject: [PATCH 072/164] Fix travis failures by removing dependency of vmware from storage. Added a new command class to verify the vCenter details provided while adding primary storage --- .../api/ValidateVcenterDetailsCommand.java | 40 +++++++++++++++++ .../vmware/resource/VmwareResource.java | 17 ++++++++ plugins/storage/volume/default/pom.xml | 6 --- ...oudStackPrimaryDataStoreLifeCycleImpl.java | 43 +++++++++++-------- 4 files changed, 83 insertions(+), 23 deletions(-) create mode 100644 core/src/main/java/com/cloud/agent/api/ValidateVcenterDetailsCommand.java diff --git a/core/src/main/java/com/cloud/agent/api/ValidateVcenterDetailsCommand.java b/core/src/main/java/com/cloud/agent/api/ValidateVcenterDetailsCommand.java new file mode 100644 index 00000000000..d1d1393efe4 --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/ValidateVcenterDetailsCommand.java @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.agent.api; + +public class ValidateVcenterDetailsCommand extends Command { + + String vCenterServerAddress; + + public ValidateVcenterDetailsCommand(String vCenterServerAddress) { + this.vCenterServerAddress = vCenterServerAddress; + } + + public String getvCenterServerAddress() { + return vCenterServerAddress; + } + + public void setvCenterServerAddress(String vCenterServerAddress) { + this.vCenterServerAddress = vCenterServerAddress; + } + + @Override + public boolean executeInSequence() { + return false; + } +} diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index bab45bf492b..eb9f333ec7a 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.hypervisor.vmware.resource; +import com.cloud.agent.api.ValidateVcenterDetailsCommand; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.storage.configdrive.ConfigDrive; import org.apache.cloudstack.storage.to.TemplateObjectTO; @@ -569,6 +570,8 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa answer = execute((GetUnmanagedInstancesCommand) cmd); } else if (clz == PrepareUnmanageVMInstanceCommand.class) { answer = execute((PrepareUnmanageVMInstanceCommand) cmd); + } else if (clz == ValidateVcenterDetailsCommand.class) { + answer = execute((ValidateVcenterDetailsCommand) cmd); } else { answer = Answer.createUnsupportedCommandAnswer(cmd); } @@ -7261,4 +7264,18 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa return new PrepareUnmanageVMInstanceAnswer(cmd, true, "OK"); } + + private Answer execute(ValidateVcenterDetailsCommand cmd) { + if (s_logger.isInfoEnabled()) { + s_logger.info("Executing resource ValidateVcenterDetailsCommand " + _gson.toJson(cmd)); + } + String vCenterServerAddress = cmd.getvCenterServerAddress(); + VmwareContext context = getServiceContext(); + + if (vCenterServerAddress.equals(context.getServerAddress())) { + return new Answer(cmd, true, "success"); + } else { + return new Answer(cmd, false, "Provided vCenter server address is invalid"); + } + } } diff --git a/plugins/storage/volume/default/pom.xml b/plugins/storage/volume/default/pom.xml index 014e7555d8c..51450e0f8a8 100644 --- a/plugins/storage/volume/default/pom.xml +++ b/plugins/storage/volume/default/pom.xml @@ -33,12 +33,6 @@ cloud-engine-storage-volume ${project.version} - - org.apache.cloudstack - cloud-plugin-hypervisor-vmware - ${project.version} - compile - diff --git a/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java b/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java index 4195b4a72a3..068b26fe70a 100644 --- a/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java +++ b/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java @@ -23,6 +23,7 @@ import com.cloud.agent.api.Answer; import com.cloud.agent.api.CreateStoragePoolCommand; import com.cloud.agent.api.DeleteStoragePoolCommand; import com.cloud.agent.api.StoragePoolInfo; +import com.cloud.agent.api.ValidateVcenterDetailsCommand; import com.cloud.alert.AlertManager; import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.StorageConflictException; @@ -30,10 +31,6 @@ import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor.HypervisorType; -import com.cloud.hypervisor.vmware.VmwareDatacenterVO; -import com.cloud.hypervisor.vmware.VmwareDatacenterZoneMapVO; -import com.cloud.hypervisor.vmware.dao.VmwareDatacenterDao; -import com.cloud.hypervisor.vmware.dao.VmwareDatacenterZoneMapDao; import com.cloud.resource.ResourceManager; import com.cloud.server.ManagementServer; import com.cloud.storage.OCFS2Manager; @@ -126,10 +123,6 @@ public class CloudStackPrimaryDataStoreLifeCycleImpl implements PrimaryDataStore StoragePoolAutomation storagePoolAutmation; @Inject protected HostDao _hostDao; - @Inject - private VmwareDatacenterZoneMapDao vmwareDatacenterZoneMapDao; - @Inject - private VmwareDatacenterDao vmwareDcDao; @SuppressWarnings("unchecked") @Override @@ -258,7 +251,7 @@ public class CloudStackPrimaryDataStoreLifeCycleImpl implements PrimaryDataStore parameters.setUserInfo(userInfo); } else if (scheme.equalsIgnoreCase("PreSetup")) { if (StringUtils.isNotBlank(hypervisorType) && HypervisorType.getType(hypervisorType).equals(HypervisorType.VMware)) { - validateVcenterDetails(zoneId, storageHost); + validateVcenterDetails(zoneId, podId, clusterId,storageHost); } parameters.setType(StoragePoolType.PreSetup); parameters.setHost(storageHost); @@ -266,7 +259,7 @@ public class CloudStackPrimaryDataStoreLifeCycleImpl implements PrimaryDataStore parameters.setPath(hostPath); } else if (scheme.equalsIgnoreCase("DatastoreCluster")) { if (StringUtils.isNotBlank(hypervisorType) && HypervisorType.getType(hypervisorType).equals(HypervisorType.VMware)) { - validateVcenterDetails(zoneId, storageHost); + validateVcenterDetails(zoneId, podId, clusterId,storageHost); } parameters.setType(StoragePoolType.DatastoreCluster); parameters.setHost(storageHost); @@ -371,15 +364,31 @@ public class CloudStackPrimaryDataStoreLifeCycleImpl implements PrimaryDataStore return dataStoreHelper.createPrimaryDataStore(parameters); } - private void validateVcenterDetails(long zoneId, String storageHost) { - VmwareDatacenterZoneMapVO vmwareDcZoneMap = vmwareDatacenterZoneMapDao.findByZoneId(zoneId); - if (vmwareDcZoneMap != null) { - Long associatedVmwareDcId = vmwareDcZoneMap.getVmwareDcId(); - VmwareDatacenterVO associatedVmwareDc = vmwareDcDao.findById(associatedVmwareDcId); - if (!associatedVmwareDc.getVcenterHost().equals(storageHost)) { - throw new InvalidParameterValueException("Provided vCenter server details does not match with the existing vCenter in zone id: " + zoneId); + private void validateVcenterDetails(Long zoneId, Long podId, Long clusterId, String storageHost) { + + List allHosts = + _resourceMgr.listAllUpHosts(Host.Type.Routing, clusterId, podId, zoneId); + if (allHosts.isEmpty()) { + throw new CloudRuntimeException("No host up to associate a storage pool with in zone: " + zoneId + " pod: " + podId + " cluster: " + clusterId); + } + + boolean success = false; + for (HostVO h : allHosts) { + ValidateVcenterDetailsCommand cmd = new ValidateVcenterDetailsCommand(storageHost); + final Answer answer = agentMgr.easySend(h.getId(), cmd); + if (answer != null && answer.getResult()) { + s_logger.info("Successfully validated vCenter details provided"); + return; + } else { + if (answer != null) { + throw new InvalidParameterValueException("Provided vCenter server details does not match with the existing vCenter in zone id: " + zoneId); + } else { + String msg = "Can not validate vCenter through host " + h.getId() + " due to ValidateVcenterDetailsCommand returns null"; + s_logger.warn(msg); + } } } + throw new CloudRuntimeException("Could not validate vCenter details through any of the hosts with in zone: " + zoneId + ", pod: " + podId + ", cluster: " + clusterId); } protected boolean createStoragePool(long hostId, StoragePool pool) { From c11dc337be774af429a1eff0f02d94e4867f28f0 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Fri, 28 Aug 2020 01:03:05 +0530 Subject: [PATCH 073/164] Set back the datastore cluster state to UP upon on failure in prepare maintenance --- .../src/main/java/com/cloud/storage/StorageManagerImpl.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index 7b00e3491c5..e08e8becc7a 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -1583,12 +1583,14 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C for (DataStore dataStore: maintenanceSuccessfulStoragePools) { lifeCycle.cancelMaintain(dataStore); } - // Set back to Up state of remaining child storage pools + // Set back to Up state of remaining child storage pools and datastore cluster while (iteratorChildDatastore.hasNext()) { StoragePoolVO childDatastore = iteratorChildDatastore.next(); childDatastore.setStatus(StoragePoolStatus.Up); _storagePoolDao.update(childDatastore.getId(), childDatastore); } + datastoreCluster.setStatus(StoragePoolStatus.Up); + _storagePoolDao.update(datastoreCluster.getId(), datastoreCluster); throw new CloudRuntimeException(String.format("Failed to prepare maintenance mode for datastore cluster %d with error %s %s", primaryStorageId, e.getMessage(), e)); } maintenanceSuccessfulStoragePools.add(childStore); From 04d400f0553c502a56863b976d316753abfb5f8d Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Fri, 28 Aug 2020 10:25:32 +0530 Subject: [PATCH 074/164] Handle import unmanaged VM into CloudStack to support any type of datastore --- .../vmware/resource/VmwareResource.java | 9 +-------- .../cloudstack/vm/UnmanagedVMsManagerImpl.java | 18 +++++++----------- 2 files changed, 8 insertions(+), 19 deletions(-) diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index eb9f333ec7a..909f5f76a78 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -299,7 +299,6 @@ import com.vmware.vim25.VirtualVmxnet2; import com.vmware.vim25.VirtualVmxnet3; import com.vmware.vim25.VmConfigInfo; import com.vmware.vim25.VmConfigSpec; -import com.vmware.vim25.VmfsDatastoreInfo; import com.vmware.vim25.VmwareDistributedVirtualSwitchPvlanSpec; import com.vmware.vim25.VmwareDistributedVirtualSwitchVlanIdSpec; import org.apache.cloudstack.storage.command.CopyCommand; @@ -7000,14 +6999,8 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa instanceDisk.setDatastorePath(dsInfo.getNas().getRemotePath()); instanceDisk.setDatastoreType(dsInfo.getNas().getType()); } - } else if (info instanceof VmfsDatastoreInfo) { - VmfsDatastoreInfo dsInfo = (VmfsDatastoreInfo) info; - instanceDisk.setDatastoreName(dsInfo.getVmfs().getName()); - instanceDisk.setDatastoreType(dsInfo.getVmfs().getType()); } else { - String msg = String.format("Unmanaged instance disk: %s is on unsupported datastore %s", instanceDisk.getDiskId(), info.getClass().getSimpleName()); - s_logger.error(msg); - throw new Exception(msg); + instanceDisk.setDatastoreName(info.getName()); } } s_logger.info(vmMo.getName() + " " + disk.getDeviceInfo().getLabel() + " " + disk.getDeviceInfo().getSummary() + " " + disk.getDiskObjectId() + " " + disk.getCapacityInKB() + " " + instanceDisk.getController()); diff --git a/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java b/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java index 0d614349235..ecf1145dfb0 100644 --- a/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java @@ -103,7 +103,6 @@ import com.cloud.service.ServiceOfferingVO; import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.storage.GuestOS; import com.cloud.storage.GuestOSHypervisor; -import com.cloud.storage.Storage; import com.cloud.storage.StoragePool; import com.cloud.storage.VMTemplateStoragePoolVO; import com.cloud.storage.VMTemplateVO; @@ -491,23 +490,20 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager { final String dsPath = disk.getDatastorePath(); final String dsType = disk.getDatastoreType(); final String dsName = disk.getDatastoreName(); - if (dsType.equals(Storage.StoragePoolType.VMFS.toString())) { - List pools = primaryDataStoreDao.listPoolsByCluster(cluster.getId()); - pools.addAll(primaryDataStoreDao.listByDataCenterId(zone.getId())); + if (dsType != null) { + List pools = primaryDataStoreDao.listPoolByHostPath(dsHost, dsPath); for (StoragePool pool : pools) { - if (pool.getPoolType() != Storage.StoragePoolType.VMFS) { - continue; - } - if (pool.getPath().endsWith(dsName)) { + if (pool.getDataCenterId() == zone.getId() && + (pool.getClusterId() == null || pool.getClusterId().equals(cluster.getId()))) { storagePool = pool; break; } } } else { - List pools = primaryDataStoreDao.listPoolByHostPath(dsHost, dsPath); + List pools = primaryDataStoreDao.listPoolsByCluster(cluster.getId()); + pools.addAll(primaryDataStoreDao.listByDataCenterId(zone.getId())); for (StoragePool pool : pools) { - if (pool.getDataCenterId() == zone.getId() && - (pool.getClusterId() == null || pool.getClusterId().equals(cluster.getId()))) { + if (pool.getPath().endsWith(dsName)) { storagePool = pool; break; } From 8ee17e19a02438d6d0cb977c31aee8f6acb005f2 Mon Sep 17 00:00:00 2001 From: davidjumani Date: Tue, 1 Sep 2020 11:03:53 +0530 Subject: [PATCH 075/164] Fix fcd from being deleted when empty --- .../com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java index 7dcc5c77f5d..3bb0d16b992 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java @@ -140,6 +140,7 @@ public class HypervisorHostHelper { private static final String VMDK_PACK_DIR = "ova"; private static final String OVA_OPTION_KEY_BOOTDISK = "cloud.ova.bootdisk"; public static final String VSPHERE_DATASTORE_BASE_FOLDER = "fcd"; + public static final String VSPHERE_DATASTORE_HIDDEN_FOLDER = ".hidden"; public static VirtualMachineMO findVmFromObjectContent(VmwareContext context, ObjectContent[] ocs, String name, String instanceNameCustomField) { @@ -2102,10 +2103,13 @@ public class HypervisorHostHelper { public static void createBaseFolderInDatastore(DatastoreMO dsMo, VmwareHypervisorHost hyperHost) throws Exception { String dsPath = String.format("[%s]", dsMo.getName()); String folderPath = String.format("[%s] %s", dsMo.getName(), VSPHERE_DATASTORE_BASE_FOLDER); + String hiddenFolderPath = String.format("%s/%s", folderPath, VSPHERE_DATASTORE_HIDDEN_FOLDER); if (!dsMo.folderExists(dsPath, VSPHERE_DATASTORE_BASE_FOLDER)) { s_logger.info(String.format("vSphere datastore base folder: %s does not exist, now creating on datastore: %s", VSPHERE_DATASTORE_BASE_FOLDER, dsMo.getName())); dsMo.makeDirectory(folderPath, hyperHost.getHyperHostDatacenter()); + // Adding another directory so vCentre doesn't remove the fcd directory when it's empty + dsMo.makeDirectory(hiddenFolderPath, hyperHost.getHyperHostDatacenter()); } } } From d864e9dc39b69e590e3df898a03a874a8173fcc6 Mon Sep 17 00:00:00 2001 From: nvazquez Date: Thu, 6 Aug 2020 14:39:23 -0300 Subject: [PATCH 076/164] [VMware] Full OVF properties support --- .../agent/api/storage/OVFConfigurationTO.java | 61 ++ ...OVFProperty.java => OVFEulaSectionTO.java} | 42 +- .../cloud/agent/api/storage/OVFHelper.java | 811 ++++++++++++++---- .../agent/api/storage/OVFPropertyTO.java | 23 +- .../api/storage/OVFVirtualHardwareItemTO.java | 365 ++++++++ .../storage/OVFVirtualHardwareSectionTO.java | 34 +- .../com/cloud/agent/api/to/DatadiskTO.java | 21 +- .../cloud/agent/api/to/DeployAsIsInfoTO.java | 51 ++ .../cloud/agent/api/to/VirtualMachineTO.java | 16 +- .../java/com/cloud/storage/ImageStore.java | 7 + .../template/VirtualMachineTemplate.java | 2 + .../main/java/com/cloud/vm/NicProfile.java | 12 + .../java/com/cloud/vm/VmDetailConstants.java | 2 + .../apache/cloudstack/api/ApiConstants.java | 7 +- .../admin/vm/ImportUnmanagedInstanceCmd.java | 19 +- .../offering/ListServiceOfferingsCmd.java | 30 + .../template/ListTemplateOVFProperties.java | 68 -- .../user/template/ListTemplatesCmd.java | 29 + .../user/template/RegisterTemplateCmd.java | 11 + .../api/command/user/vm/DeployVMCmd.java | 55 +- .../api/net/NetworkPrerequisiteTO.java | 124 +++ .../response/TemplateOVFPropertyResponse.java | 43 +- .../api/response/TemplateResponse.java | 34 +- .../apache/cloudstack/query/QueryService.java | 4 - .../agent/api/storage/OVFHelperTest.java | 677 ++++++++++++++- core/pom.xml | 4 - .../agent/api/storage/DownloadAnswer.java | 43 + .../api/storage/GetDatadisksCommand.java | 7 +- .../com/cloud/resource/ServerResource.java | 1 + .../storage/resource/StorageProcessor.java | 3 + .../cloud/storage/template/OVAProcessor.java | 200 +++-- .../com/cloud/storage/template/Processor.java | 10 +- .../storage/to/TemplateObjectTO.java | 16 + .../agent/api/storage/DownloadAnswerTest.java | 58 ++ .../service/VolumeOrchestrationService.java | 8 +- .../service/api/OrchestrationService.java | 6 +- .../api/storage/PrimaryDataStore.java | 6 +- .../api/storage/TemplateDataFactory.java | 4 +- .../subsystem/api/storage/TemplateInfo.java | 4 + .../api/storage/TemplateService.java | 7 +- .../subsystem/api/storage/VolumeInfo.java | 4 + .../image/datastore/ImageStoreEntity.java | 2 +- .../com/cloud/template/TemplateManager.java | 3 + .../cloud/vm/VirtualMachineManagerImpl.java | 13 +- .../orchestration/CloudOrchestrator.java | 9 +- .../orchestration/NetworkOrchestrator.java | 279 +++--- .../orchestration/VolumeOrchestrator.java | 111 ++- .../cloud/storage/TemplateOVFPropertyVO.java | 167 ---- .../com/cloud/storage/VMTemplateDetailVO.java | 4 +- .../storage/VMTemplateStoragePoolVO.java | 16 +- .../java/com/cloud/storage/VMTemplateVO.java | 17 +- .../dao/TemplateOVFPropertiesDaoImpl.java | 78 -- .../storage/dao/VMTemplateDetailsDao.java | 14 + .../storage/dao/VMTemplateDetailsDaoImpl.java | 126 ++- .../cloud/storage/dao/VMTemplatePoolDao.java | 20 +- .../storage/dao/VMTemplatePoolDaoImpl.java | 23 +- .../java/com/cloud/vm/dao/UserVmDaoImpl.java | 15 +- ...spring-engine-schema-core-daos-context.xml | 1 - .../META-INF/db/schema-41400to41500.sql | 214 ++++- .../manager/StorageCacheManagerImpl.java | 2 +- ...vmNonManagedStorageDataMotionStrategy.java | 6 +- .../StorageSystemDataMotionStrategy.java | 8 +- ...NonManagedStorageSystemDataMotionTest.java | 9 +- .../image/TemplateDataFactoryImpl.java | 35 +- .../storage/image/TemplateServiceImpl.java | 60 +- .../storage/image/store/ImageStoreImpl.java | 4 +- .../storage/image/store/TemplateObject.java | 21 +- .../datastore/DataObjectManagerImpl.java | 2 +- .../datastore/ObjectInDataStoreManager.java | 4 +- .../ObjectInDataStoreManagerImpl.java | 22 +- .../image/BaseImageStoreDriverImpl.java | 290 +++++-- .../storage/image/ImageStoreDriver.java | 2 +- .../storage/image/TemplateEntityImpl.java | 314 ------- .../datastore/PrimaryDataStoreImpl.java | 28 +- .../storage/volume/VolumeObject.java | 31 +- .../storage/volume/VolumeServiceImpl.java | 59 +- .../com/cloud/hypervisor/guru/VMwareGuru.java | 4 +- .../hypervisor/guru/VmwareVmImplementer.java | 137 +-- .../manager/VmwareStorageManagerImpl.java | 4 +- .../vmware/resource/VmwareResource.java | 183 +++- .../resource/VmwareStorageProcessor.java | 193 +++-- .../driver/DateraPrimaryDataStoreDriver.java | 6 +- .../SolidFirePrimaryDataStoreDriver.java | 4 +- .../main/java/com/cloud/api/ApiDBUtils.java | 8 +- .../java/com/cloud/api/ApiResponseHelper.java | 4 +- .../com/cloud/api/query/QueryManagerImpl.java | 97 +-- .../cloud/api/query/ViewResponseHelper.java | 11 +- .../cloud/api/query/dao/TemplateJoinDao.java | 6 +- .../api/query/dao/TemplateJoinDaoImpl.java | 32 +- .../api/query/dao/UserVmJoinDaoImpl.java | 4 +- .../api/query/vo/ServiceOfferingJoinVO.java | 28 + .../cloud/api/query/vo/TemplateJoinVO.java | 7 + .../cloud/server/ManagementServerImpl.java | 2 - .../storage/ImageStoreUploadMonitorImpl.java | 4 +- .../com/cloud/storage/TemplateProfile.java | 8 +- .../storage/upload/params/UploadParams.java | 1 + .../upload/params/UploadParamsBase.java | 5 + .../template/HypervisorTemplateAdapter.java | 20 +- .../com/cloud/template/TemplateAdapter.java | 13 +- .../cloud/template/TemplateAdapterBase.java | 17 +- .../cloud/template/TemplateManagerImpl.java | 21 +- .../java/com/cloud/vm/UserVmManagerImpl.java | 303 +++++-- .../download/DirectDownloadManagerImpl.java | 4 +- .../query/dao/TemplateJoinDaoImplTest.java | 1 - .../template/TemplateManagerImplTest.java | 13 +- .../vm/DeploymentPlanningManagerImplTest.java | 14 + .../resource/NfsSecondaryStorageResource.java | 10 +- .../storage/template/DownloadManagerImpl.java | 62 +- test/integration/smoke/test_vm_life_cycle.py | 280 +++++- tools/marvin/marvin/config/test_data.py | 55 +- tools/marvin/marvin/lib/base.py | 12 +- tools/marvin/marvin/lib/common.py | 68 ++ tools/marvin/marvin/lib/vcenter.py | 6 + ui/l10n/en.js | 1 + ui/scripts/docs.js | 4 + ui/scripts/instanceWizard.js | 24 +- ui/scripts/templates.js | 97 +-- ui/scripts/ui-custom/instanceWizard.js | 16 +- utils/pom.xml | 4 + .../utils/compression/CompressionUtil.java | 58 ++ .../compression/CompressionUtilTest.java | 128 +++ .../cloud/hypervisor/vmware/mo/ClusterMO.java | 4 +- .../cloud/hypervisor/vmware/mo/HostMO.java | 8 +- .../vmware/mo/HypervisorHostHelper.java | 144 +++- .../vmware/mo/VirtualMachineMO.java | 26 + .../vmware/mo/VmwareHypervisorHost.java | 2 +- .../hypervisor/vmware/util/VmwareContext.java | 32 +- 127 files changed, 5346 insertions(+), 1787 deletions(-) create mode 100644 api/src/main/java/com/cloud/agent/api/storage/OVFConfigurationTO.java rename api/src/main/java/com/cloud/agent/api/storage/{OVFProperty.java => OVFEulaSectionTO.java} (55%) create mode 100644 api/src/main/java/com/cloud/agent/api/storage/OVFVirtualHardwareItemTO.java rename engine/schema/src/main/java/com/cloud/storage/dao/TemplateOVFPropertiesDao.java => api/src/main/java/com/cloud/agent/api/storage/OVFVirtualHardwareSectionTO.java (54%) create mode 100644 api/src/main/java/com/cloud/agent/api/to/DeployAsIsInfoTO.java delete mode 100644 api/src/main/java/org/apache/cloudstack/api/command/user/template/ListTemplateOVFProperties.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/net/NetworkPrerequisiteTO.java create mode 100644 core/src/test/java/com/cloud/agent/api/storage/DownloadAnswerTest.java delete mode 100644 engine/schema/src/main/java/com/cloud/storage/TemplateOVFPropertyVO.java delete mode 100644 engine/schema/src/main/java/com/cloud/storage/dao/TemplateOVFPropertiesDaoImpl.java delete mode 100644 engine/storage/src/main/java/org/apache/cloudstack/storage/image/TemplateEntityImpl.java create mode 100644 utils/src/main/java/com/cloud/utils/compression/CompressionUtil.java create mode 100644 utils/src/test/java/com/cloud/utils/compression/CompressionUtilTest.java diff --git a/api/src/main/java/com/cloud/agent/api/storage/OVFConfigurationTO.java b/api/src/main/java/com/cloud/agent/api/storage/OVFConfigurationTO.java new file mode 100644 index 00000000000..983a8b808f4 --- /dev/null +++ b/api/src/main/java/com/cloud/agent/api/storage/OVFConfigurationTO.java @@ -0,0 +1,61 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package com.cloud.agent.api.storage; + +import java.util.List; + +public class OVFConfigurationTO { + + private final String id; + private final String label; + private final String description; + private List hardwareItems; + private int index; + + public OVFConfigurationTO(String id, String label, String description, int index) { + this.id = id; + this.label = label; + this.description = description; + this.index = index; + } + + public String getId() { + return id; + } + + public String getLabel() { + return label; + } + + public String getDescription() { + return description; + } + + public void setHardwareItems(List items) { + this.hardwareItems = items; + } + + public List getHardwareItems() { + return hardwareItems; + } + + public int getIndex() { + return index; + } +} \ No newline at end of file diff --git a/api/src/main/java/com/cloud/agent/api/storage/OVFProperty.java b/api/src/main/java/com/cloud/agent/api/storage/OVFEulaSectionTO.java similarity index 55% rename from api/src/main/java/com/cloud/agent/api/storage/OVFProperty.java rename to api/src/main/java/com/cloud/agent/api/storage/OVFEulaSectionTO.java index ac9ae7721b0..eff478c88b3 100644 --- a/api/src/main/java/com/cloud/agent/api/storage/OVFProperty.java +++ b/api/src/main/java/com/cloud/agent/api/storage/OVFEulaSectionTO.java @@ -16,18 +16,36 @@ // specific language governing permissions and limitations // under the License. // - package com.cloud.agent.api.storage; -public interface OVFProperty { +import com.cloud.agent.api.LogLevel; - Long getTemplateId(); - String getKey(); - String getType(); - String getValue(); - String getQualifiers(); - Boolean isUserConfigurable(); - String getLabel(); - String getDescription(); - Boolean isPassword(); -} \ No newline at end of file +import java.io.Serializable; + +/** + * End-user licence agreement + */ +public class OVFEulaSectionTO implements Serializable { + private String info; + @LogLevel(LogLevel.Log4jLevel.Off) + private byte[] compressedLicense; + private int index; + + public OVFEulaSectionTO(String info, byte[] license, int eulaIndex) { + this.info = info; + this.compressedLicense = license; + this.index = eulaIndex; + } + + public String getInfo() { + return this.info; + } + + public byte[] getCompressedLicense() { + return this.compressedLicense; + } + + public int getIndex() { + return index; + } +} diff --git a/api/src/main/java/com/cloud/agent/api/storage/OVFHelper.java b/api/src/main/java/com/cloud/agent/api/storage/OVFHelper.java index 15d63587490..bd58a189f76 100644 --- a/api/src/main/java/com/cloud/agent/api/storage/OVFHelper.java +++ b/api/src/main/java/com/cloud/agent/api/storage/OVFHelper.java @@ -22,8 +22,15 @@ import java.io.PrintWriter; import java.io.StringReader; import java.io.StringWriter; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.Transformer; @@ -34,13 +41,20 @@ import javax.xml.transform.stream.StreamResult; import com.cloud.configuration.Resource.ResourceType; import com.cloud.exception.InternalErrorException; +import com.cloud.utils.compression.CompressionUtil; +import org.apache.cloudstack.api.net.NetworkPrerequisiteTO; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.math.NumberUtils; import org.apache.log4j.Logger; import org.w3c.dom.Document; import org.w3c.dom.Element; +import org.w3c.dom.NamedNodeMap; import org.w3c.dom.Node; import org.w3c.dom.NodeList; +import org.w3c.dom.traversal.DocumentTraversal; +import org.w3c.dom.traversal.NodeFilter; +import org.w3c.dom.traversal.NodeIterator; import org.xml.sax.InputSource; import org.xml.sax.SAXException; @@ -73,10 +87,11 @@ public class OVFHelper { } /** - * Get the text value of a node's child with name "childNodeName", null if not present + * Get the text value of a node's child with name or suffix "childNodeName", null if not present * Example: * * Text value + * Text value * */ private String getChildNodeValue(Node node, String childNodeName) { @@ -84,7 +99,9 @@ public class OVFHelper { NodeList childNodes = node.getChildNodes(); for (int i = 0; i < childNodes.getLength(); i++) { Node value = childNodes.item(i); - if (value != null && value.getNodeName().equals(childNodeName)) { + // Also match if the child's name has a suffix: + // Example: + if (value != null && (value.getNodeName().equals(childNodeName)) || value.getNodeName().endsWith(":" + childNodeName)) { return value.getTextContent(); } } @@ -92,174 +109,320 @@ public class OVFHelper { return null; } + /** + * Check if the attribute is present on the element, otherwise check preprending ':' + */ + private String getNodeAttribute(Element element, String prefix, String attr) { + if (element == null) { + return null; + } + if (element.hasAttribute(prefix + ":" + attr)) { + return element.getAttribute(prefix + ":" + attr); + } + else if (element.hasAttribute(attr)) { + return element.getAttribute(attr); + } + + NamedNodeMap attributes = element.getAttributes(); + if (attributes == null || attributes.getLength() == 0) { + return null; + } + for (int i = 0; i < attributes.getLength(); i++) { + Node node = attributes.item(i); + if (node != null && node.getNodeName().endsWith(":" + attr)) { + return node.getTextContent(); + } + } + return null; + } + /** * Create OVFProperty class from the parsed node. Note that some fields may not be present. * The key attribute is required */ - protected OVFPropertyTO createOVFPropertyFromNode(Node node) { - Element property = (Element) node; - String key = property.getAttribute("ovf:key"); + protected OVFPropertyTO createOVFPropertyFromNode(Node node, int index, String category) { + Element element = (Element) node; + String key = getNodeAttribute(element, "ovf","key"); if (StringUtils.isBlank(key)) { return null; } - String value = property.getAttribute("ovf:value"); - String type = property.getAttribute("ovf:type"); - String qualifiers = property.getAttribute("ovf:qualifiers"); - String userConfigurableStr = property.getAttribute("ovf:userConfigurable"); + String value = getNodeAttribute(element, "ovf","value"); + String type = getNodeAttribute(element, "ovf","type"); + String qualifiers = getNodeAttribute(element, "ovf","qualifiers"); + String userConfigurableStr = getNodeAttribute(element, "ovf","userConfigurable"); boolean userConfigurable = StringUtils.isNotBlank(userConfigurableStr) && userConfigurableStr.equalsIgnoreCase("true"); - String passStr = property.getAttribute("ovf:password"); + String passStr = getNodeAttribute(element, "ovf","password"); boolean password = StringUtils.isNotBlank(passStr) && passStr.equalsIgnoreCase("true"); String label = getChildNodeValue(node, "Label"); String description = getChildNodeValue(node, "Description"); - return new OVFPropertyTO(key, type, value, qualifiers, userConfigurable, label, description, password); + s_logger.debug("Creating OVF property index " + index + (category == null ? "" : " for category " + category) + + " with key = " + key); + return new OVFPropertyTO(key, type, value, qualifiers, userConfigurable, + label, description, password, index, category); } /** - * Retrieve OVF properties from a parsed OVF file, with attribute 'ovf:userConfigurable' set to true + * Retrieve OVF properties from a parsed OVF file including its category (if available) and in-order, + * with attribute 'ovf:userConfigurable' set to true. */ - private List getConfigurableOVFPropertiesFromDocument(Document doc) { + public List getConfigurableOVFPropertiesFromDocument(Document doc) { List props = new ArrayList<>(); - NodeList properties = doc.getElementsByTagName("Property"); - if (properties != null) { - for (int i = 0; i < properties.getLength(); i++) { - Node node = properties.item(i); + if (doc == null) { + return props; + } + int propertyIndex = 0; + NodeList productSections = doc.getElementsByTagName("ProductSection"); + if (productSections != null) { + String lastCategoryFound = null; + for (int i = 0; i < productSections.getLength(); i++) { + Node node = productSections.item(i); if (node == null) { continue; } - OVFPropertyTO prop = createOVFPropertyFromNode(node); - if (prop != null && prop.isUserConfigurable()) { - props.add(prop); + NodeList childNodes = node.getChildNodes(); + for (int j = 0; j < childNodes.getLength(); j++) { + Node child = childNodes.item(j); + if (child == null) { + continue; + } + if (child.getNodeName().equalsIgnoreCase("Category")) { + lastCategoryFound = child.getTextContent(); + s_logger.info("Category found " + lastCategoryFound); + } else if (child.getNodeName().equalsIgnoreCase("Property")) { + OVFPropertyTO prop = createOVFPropertyFromNode(child, propertyIndex, lastCategoryFound); + if (prop != null && prop.isUserConfigurable()) { + props.add(prop); + propertyIndex++; + } + } } } } return props; } - /** - * Get properties from OVF file located on ovfFilePath - */ - public List getOVFPropertiesFromFile(String ovfFilePath) throws ParserConfigurationException, IOException, SAXException { - if (StringUtils.isBlank(ovfFilePath)) { - return new ArrayList<>(); - } - File ovfFile = new File(ovfFilePath); - final Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(ovfFile); - return getConfigurableOVFPropertiesFromDocument(doc); - } - /** * Get properties from OVF XML string */ - protected List getOVFPropertiesXmlString(final String ovfFilePath) throws ParserConfigurationException, IOException, SAXException { - InputSource is = new InputSource(new StringReader(ovfFilePath)); + protected List getOVFPropertiesFromXmlString(final String ovfString) throws ParserConfigurationException, IOException, SAXException { + InputSource is = new InputSource(new StringReader(ovfString)); final Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(is); return getConfigurableOVFPropertiesFromDocument(doc); } - public List getOVFVolumeInfo(final String ovfFilePath) { + protected List getOVFDeploymentOptionsFromXmlString(final String ovfString) throws ParserConfigurationException, IOException, SAXException { + InputSource is = new InputSource(new StringReader(ovfString)); + final Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(is); + return getDeploymentOptionsFromDocumentTree(doc); + } + + protected List getOVFVirtualHardwareSectionFromXmlString(final String ovfString) throws ParserConfigurationException, IOException, SAXException { + InputSource is = new InputSource(new StringReader(ovfString)); + final Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(is); + return getVirtualHardwareItemsFromDocumentTree(doc); + } + + protected List getOVFEulaSectionFromXmlString(final String ovfString) throws ParserConfigurationException, IOException, SAXException { + InputSource is = new InputSource(new StringReader(ovfString)); + final Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(is); + return getEulaSectionsFromDocument(doc); + } + + public List getOVFVolumeInfoFromFile(final String ovfFilePath, final String configurationId) throws InternalErrorException { if (StringUtils.isBlank(ovfFilePath)) { - return new ArrayList(); + return new ArrayList<>(); + } + Document doc = getDocumentFromFile(ovfFilePath); + + return getOVFVolumeInfoFromFile(ovfFilePath, doc, configurationId); + } + + public List getOVFVolumeInfoFromFile(String ovfFilePath, Document doc, String configurationId) throws InternalErrorException { + if (org.apache.commons.lang.StringUtils.isBlank(ovfFilePath)) { + return null; } - ArrayList vf = new ArrayList(); - ArrayList vd = new ArrayList(); File ovfFile = new File(ovfFilePath); + List hardwareItems = getVirtualHardwareItemsFromDocumentTree(doc); + List files = extractFilesFromOvfDocumentTree(ovfFile, doc); + List disks = extractDisksFromOvfDocumentTree(doc); + + List diskHardwareItems = hardwareItems.stream() + .filter(x -> x.getResourceType() == OVFVirtualHardwareItemTO.HardwareResourceType.DiskDrive && + hardwareItemContainsConfiguration(x, configurationId)) + .collect(Collectors.toList()); + List diskTOs = matchHardwareItemsToDiskAndFilesInformation(diskHardwareItems, files, disks, ovfFile.getParent()); + return diskTOs; + } + + private String extractDiskIdFromDiskHostResource(String hostResource) { + if (hostResource.startsWith("ovf:/disk/")) { + return hostResource.replace("ovf:/disk/", ""); + } + String[] resourceParts = hostResource.split("/"); + return resourceParts[resourceParts.length - 1]; + } + + private OVFDisk getDiskDefinitionFromDiskId(String diskId, List disks) { + for (OVFDisk disk : disks) { + if (disk._diskId.equalsIgnoreCase(diskId)) { + return disk; + } + } + return null; + } + + private List matchHardwareItemsToDiskAndFilesInformation(List diskHardwareItems, + List files, List disks, + String ovfParentPath) throws InternalErrorException { + List diskTOs = new LinkedList<>(); + int diskNumber = 0; + for (OVFVirtualHardwareItemTO diskItem : diskHardwareItems) { + if (StringUtils.isBlank(diskItem.getHostResource())) { + s_logger.error("Missing disk information for hardware item " + diskItem.getElementName() + " " + diskItem.getInstanceId()); + continue; + } + String diskId = extractDiskIdFromDiskHostResource(diskItem.getHostResource()); + OVFDisk diskDefinition = getDiskDefinitionFromDiskId(diskId, disks); + if (diskDefinition == null) { + s_logger.error("Missing disk definition for disk ID " + diskId); + } + OVFFile fileDefinition = getFileDefinitionFromDiskDefinition(diskDefinition._fileRef, files); + DatadiskTO datadiskTO = generateDiskTO(fileDefinition, diskDefinition, ovfParentPath, diskNumber, diskItem); + diskTOs.add(datadiskTO); + diskNumber++; + } + List isoFiles = files.stream().filter(x -> x.isIso).collect(Collectors.toList()); + for (OVFFile isoFile : isoFiles) { + DatadiskTO isoTO = generateDiskTO(isoFile, null, ovfParentPath, diskNumber, null); + diskTOs.add(isoTO); + diskNumber++; + } + return diskTOs; + } + + private DatadiskTO generateDiskTO(OVFFile file, OVFDisk disk, String ovfParentPath, int diskNumber, + OVFVirtualHardwareItemTO diskItem) throws InternalErrorException { + String path = file != null ? ovfParentPath + File.separator + file._href : null; + if (StringUtils.isNotBlank(path)) { + File f = new File(path); + if (!f.exists() || f.isDirectory()) { + s_logger.error("One of the attached disk or iso does not exists " + path); + throw new InternalErrorException("One of the attached disk or iso as stated on OVF does not exists " + path); + } + } + Long capacity = disk != null ? disk._capacity : file._size; + Long fileSize = file != null ? file._size : 0L; + + String controller = ""; + String controllerSubType = ""; + if (disk != null) { + OVFDiskController cDiskController = disk._controller; + controller = cDiskController == null ? "" : disk._controller._name; + controllerSubType = cDiskController == null ? "" : disk._controller._subType; + } + + boolean isIso = file != null && file.isIso; + boolean bootable = file != null && file._bootable; + String diskId = disk == null ? file._id : disk._diskId; + String configuration = diskItem != null ? diskItem.getConfigurationIds() : null; + return new DatadiskTO(path, capacity, fileSize, diskId, + isIso, bootable, controller, controllerSubType, diskNumber, configuration); + } + + protected List extractDisksFromOvfDocumentTree(Document doc) { + NodeList disks = doc.getElementsByTagName("Disk"); + NodeList ovfDisks = doc.getElementsByTagName("ovf:Disk"); + NodeList items = doc.getElementsByTagName("Item"); + + int totalDisksLength = disks.getLength() + ovfDisks.getLength(); + ArrayList vd = new ArrayList<>(); + for (int i = 0; i < totalDisksLength; i++) { + Element disk; + if (i >= disks.getLength()) { + int pos = i - disks.getLength(); + disk = (Element) ovfDisks.item(pos); + } else { + disk = (Element) disks.item(i); + } + + if (disk == null) { + continue; + } + OVFDisk od = new OVFDisk(); + String virtualSize = getNodeAttribute(disk, "ovf", "capacity"); + od._capacity = NumberUtils.toLong(virtualSize, 0L); + String allocationUnits = getNodeAttribute(disk,"ovf","capacityAllocationUnits"); + od._diskId = getNodeAttribute(disk,"ovf","diskId"); + od._fileRef = getNodeAttribute(disk,"ovf","fileRef"); + od._populatedSize = NumberUtils.toLong(getNodeAttribute(disk,"ovf","populatedSize")); + + if ((od._capacity != 0) && (allocationUnits != null)) { + long units = 1; + if (allocationUnits.equalsIgnoreCase("KB") || allocationUnits.equalsIgnoreCase("KiloBytes") || allocationUnits.equalsIgnoreCase("byte * 2^10")) { + units = ResourceType.bytesToKiB; + } else if (allocationUnits.equalsIgnoreCase("MB") || allocationUnits.equalsIgnoreCase("MegaBytes") || allocationUnits.equalsIgnoreCase("byte * 2^20")) { + units = ResourceType.bytesToMiB; + } else if (allocationUnits.equalsIgnoreCase("GB") || allocationUnits.equalsIgnoreCase("GigaBytes") || allocationUnits.equalsIgnoreCase("byte * 2^30")) { + units = ResourceType.bytesToGiB; + } + od._capacity = od._capacity * units; + } + od._controller = getControllerType(items, od._diskId); + vd.add(od); + } + if (s_logger.isTraceEnabled()) { + s_logger.trace(String.format("found %d disk definitions",vd.size())); + } + return vd; + } + + protected List extractFilesFromOvfDocumentTree(File ovfFile, Document doc) { + NodeList files = doc.getElementsByTagName("File"); + ArrayList vf = new ArrayList<>(); + boolean toggle = true; + for (int j = 0; j < files.getLength(); j++) { + Element file = (Element)files.item(j); + OVFFile of = new OVFFile(); + of._href = getNodeAttribute(file,"ovf","href"); + if (of._href.endsWith("vmdk") || of._href.endsWith("iso")) { + of._id = getNodeAttribute(file,"ovf","id"); + String size = getNodeAttribute(file,"ovf", "size"); + if (StringUtils.isNotBlank(size)) { + of._size = Long.parseLong(size); + } else { + String dataDiskPath = ovfFile.getParent() + File.separator + of._href; + File this_file = new File(dataDiskPath); + of._size = this_file.length(); + } + of.isIso = of._href.endsWith("iso"); + if (toggle && !of.isIso) { + of._bootable = true; + toggle = !toggle; + } + vf.add(of); + } + } + if (s_logger.isTraceEnabled()) { + s_logger.trace(String.format("found %d file definitions in %s",vf.size(), ovfFile.getPath())); + } + return vf; + } + + public Document getDocumentFromFile(String ovfFilePath) { + if (org.apache.commons.lang.StringUtils.isBlank(ovfFilePath)) { + return null; + } + DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newDefaultInstance(); try { - final Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(new File(ovfFilePath)); - NodeList disks = doc.getElementsByTagName("Disk"); - NodeList files = doc.getElementsByTagName("File"); - NodeList items = doc.getElementsByTagName("Item"); - boolean toggle = true; - for (int j = 0; j < files.getLength(); j++) { - Element file = (Element)files.item(j); - OVFFile of = new OVFFile(); - of._href = file.getAttribute("ovf:href"); - if (of._href.endsWith("vmdk") || of._href.endsWith("iso")) { - of._id = file.getAttribute("ovf:id"); - String size = file.getAttribute("ovf:size"); - if (StringUtils.isNotBlank(size)) { - of._size = Long.parseLong(size); - } else { - String dataDiskPath = ovfFile.getParent() + File.separator + of._href; - File this_file = new File(dataDiskPath); - of._size = this_file.length(); - } - of.isIso = of._href.endsWith("iso"); - if (toggle && !of.isIso) { - of._bootable = true; - toggle = !toggle; - } - vf.add(of); - } - } - for (int i = 0; i < disks.getLength(); i++) { - Element disk = (Element)disks.item(i); - OVFDisk od = new OVFDisk(); - String virtualSize = disk.getAttribute("ovf:capacity"); - od._capacity = NumberUtils.toLong(virtualSize, 0L); - String allocationUnits = disk.getAttribute("ovf:capacityAllocationUnits"); - od._diskId = disk.getAttribute("ovf:diskId"); - od._fileRef = disk.getAttribute("ovf:fileRef"); - od._populatedSize = NumberUtils.toLong(disk.getAttribute("ovf:populatedSize")); - - if ((od._capacity != 0) && (allocationUnits != null)) { - - long units = 1; - if (allocationUnits.equalsIgnoreCase("KB") || allocationUnits.equalsIgnoreCase("KiloBytes") || allocationUnits.equalsIgnoreCase("byte * 2^10")) { - units = ResourceType.bytesToKiB; - } else if (allocationUnits.equalsIgnoreCase("MB") || allocationUnits.equalsIgnoreCase("MegaBytes") || allocationUnits.equalsIgnoreCase("byte * 2^20")) { - units = ResourceType.bytesToMiB; - } else if (allocationUnits.equalsIgnoreCase("GB") || allocationUnits.equalsIgnoreCase("GigaBytes") || allocationUnits.equalsIgnoreCase("byte * 2^30")) { - units = ResourceType.bytesToGiB; - } - od._capacity = od._capacity * units; - } - od._controller = getControllerType(items, od._diskId); - vd.add(od); - } - + DocumentBuilder builder = documentBuilderFactory.newDocumentBuilder(); + return builder.parse(new File(ovfFilePath)); } catch (SAXException | IOException | ParserConfigurationException e) { s_logger.error("Unexpected exception caught while parsing ovf file:" + ovfFilePath, e); throw new CloudRuntimeException(e); } - - List disksTO = new ArrayList(); - for (OVFFile of : vf) { - if (StringUtils.isBlank(of._id)){ - s_logger.error("The ovf file info is incomplete file info"); - throw new CloudRuntimeException("The ovf file info has incomplete file info"); - } - OVFDisk cdisk = getDisk(of._id, vd); - if (cdisk == null && !of.isIso){ - s_logger.error("The ovf file info has incomplete disk info"); - throw new CloudRuntimeException("The ovf file info has incomplete disk info"); - } - Long capacity = cdisk == null ? of._size : cdisk._capacity; - String controller = ""; - String controllerSubType = ""; - if (cdisk != null) { - OVFDiskController cDiskController = cdisk._controller; - controller = cDiskController == null ? "" : cdisk._controller._name; - controllerSubType = cDiskController == null ? "" : cdisk._controller._subType; - } - - String dataDiskPath = ovfFile.getParent() + File.separator + of._href; - File f = new File(dataDiskPath); - if (!f.exists() || f.isDirectory()) { - s_logger.error("One of the attached disk or iso does not exists " + dataDiskPath); - throw new CloudRuntimeException("One of the attached disk or iso as stated on OVF does not exists " + dataDiskPath); - } - disksTO.add(new DatadiskTO(dataDiskPath, capacity, of._size, of._id, of.isIso, of._bootable, controller, controllerSubType)); - } - //check if first disk is an iso move it to the end - DatadiskTO fd = disksTO.get(0); - if (fd.isIso()) { - disksTO.remove(0); - disksTO.add(fd); - } - return disksTO; } private OVFDiskController getControllerType(final NodeList itemList, final String diskId) { @@ -330,55 +493,61 @@ public class OVFHelper { return dc; } - public void rewriteOVFFile(final String origOvfFilePath, final String newOvfFilePath, final String diskName) { - try { - final Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(new File(origOvfFilePath)); - NodeList disks = doc.getElementsByTagName("Disk"); - NodeList files = doc.getElementsByTagName("File"); - NodeList items = doc.getElementsByTagName("Item"); - String keepfile = null; - List toremove = new ArrayList(); - for (int j = 0; j < files.getLength(); j++) { - Element file = (Element)files.item(j); - String href = file.getAttribute("ovf:href"); - if (diskName.equals(href)) { - keepfile = file.getAttribute("ovf:id"); - } else { - toremove.add(file); - } + public void rewriteOVFFileForSingleDisk(final String origOvfFilePath, final String newOvfFilePath, final String diskName) { + final Document doc = getDocumentFromFile(origOvfFilePath); + + NodeList disks = doc.getElementsByTagName("Disk"); + NodeList files = doc.getElementsByTagName("File"); + NodeList items = doc.getElementsByTagName("Item"); + String keepfile = null; + List toremove = new ArrayList<>(); + for (int j = 0; j < files.getLength(); j++) { + Element file = (Element)files.item(j); + String href = getNodeAttribute(file,"ovf", "href"); + if (diskName.equals(href)) { + keepfile = getNodeAttribute(file,"ovf","id"); + } else { + toremove.add(file); } - String keepdisk = null; - for (int i = 0; i < disks.getLength(); i++) { - Element disk = (Element)disks.item(i); - String fileRef = disk.getAttribute("ovf:fileRef"); - if (keepfile == null) { - s_logger.info("FATAL: OVA format error"); - } else if (keepfile.equals(fileRef)) { - keepdisk = disk.getAttribute("ovf:diskId"); - } else { - toremove.add(disk); - } + } + String keepdisk = null; + for (int i = 0; i < disks.getLength(); i++) { + Element disk = (Element)disks.item(i); + String fileRef = getNodeAttribute(disk,"ovf","fileRef"); + if (keepfile == null) { + s_logger.info("FATAL: OVA format error"); + } else if (keepfile.equals(fileRef)) { + keepdisk = getNodeAttribute(disk,"ovf","diskId"); + } else { + toremove.add(disk); } - for (int k = 0; k < items.getLength(); k++) { - Element item = (Element)items.item(k); - NodeList cn = item.getChildNodes(); - for (int l = 0; l < cn.getLength(); l++) { - if (cn.item(l) instanceof Element) { - Element el = (Element)cn.item(l); - if ("rasd:HostResource".equals(el.getNodeName()) - && !(el.getTextContent().contains("ovf:/file/" + keepdisk) || el.getTextContent().contains("ovf:/disk/" + keepdisk))) { - toremove.add(item); - break; - } + } + for (int k = 0; k < items.getLength(); k++) { + Element item = (Element) items.item(k); + NodeList cn = item.getChildNodes(); + for (int l = 0; l < cn.getLength(); l++) { + if (cn.item(l) instanceof Element) { + Element el = (Element) cn.item(l); + if ("rasd:HostResource".equals(el.getNodeName()) + && !(el.getTextContent().contains("ovf:/file/" + keepdisk) || el.getTextContent().contains("ovf:/disk/" + keepdisk))) { + toremove.add(item); + break; } } } + } - for (Element rme : toremove) { - if (rme.getParentNode() != null) { - rme.getParentNode().removeChild(rme); - } + for (Element rme : toremove) { + if (rme.getParentNode() != null) { + rme.getParentNode().removeChild(rme); } + } + + writeDocumentToFile(newOvfFilePath, doc); + } + + private void writeDocumentToFile(String newOvfFilePath, Document doc) { + try { final StringWriter writer = new StringWriter(); final StreamResult result = new StreamResult(writer); @@ -389,21 +558,312 @@ public class OVFHelper { PrintWriter outfile = new PrintWriter(newOvfFilePath); outfile.write(writer.toString()); outfile.close(); - } catch (SAXException | IOException | ParserConfigurationException | TransformerException e) { - s_logger.info("Unexpected exception caught while removing network elements from OVF:" + e.getMessage(), e); + } catch (IOException | TransformerException e) { + s_logger.info("Unexpected exception caught while rewriting OVF:" + e.getMessage(), e); throw new CloudRuntimeException(e); } } - OVFDisk getDisk(String fileRef, List disks) { - for (OVFDisk disk : disks) { - if (disk._fileRef.equals(fileRef)) { - return disk; + OVFFile getFileDefinitionFromDiskDefinition(String fileRef, List files) { + for (OVFFile file : files) { + if (file._id.equals(fileRef)) { + return file; } } return null; } + public List getNetPrerequisitesFromDocument(Document doc) throws InternalErrorException { + if (doc == null) { + if (s_logger.isTraceEnabled()) { + s_logger.trace("no document to parse; returning no prerequiste networks"); + } + return Collections.emptyList(); + } + + Map nets = getNetworksFromDocumentTree(doc); + + checkForOnlyOneSystemNode(doc); + + matchNicsToNets(nets, doc); + + return new ArrayList<>(nets.values()); + } + + private void matchNicsToNets(Map nets, Node systemElement) { + final DocumentTraversal traversal = (DocumentTraversal) systemElement; + final NodeIterator iterator = traversal.createNodeIterator(systemElement, NodeFilter.SHOW_ELEMENT, null, true); + if (s_logger.isTraceEnabled()) { + s_logger.trace(String.format("starting out with %d network-prerequisites, parsing hardware",nets.size())); + } + int nicCount = 0; + for (Node n = iterator.nextNode(); n != null; n = iterator.nextNode()) { + final Element e = (Element) n; + if ("rasd:Connection".equals(e.getTagName())) { + nicCount++; + String name = e.getTextContent(); // should be in our nets + if(nets.get(name) == null) { + if(s_logger.isInfoEnabled()) { + s_logger.info(String.format("found a nic definition without a network definition byname %s, adding it to the list.", name)); + } + nets.put(name, new NetworkPrerequisiteTO()); + } + NetworkPrerequisiteTO thisNet = nets.get(name); + if (e.getParentNode() != null) { + fillNicPrerequisites(thisNet,e.getParentNode()); + } + } + } + if (s_logger.isTraceEnabled()) { + s_logger.trace(String.format("ending up with %d network-prerequisites, parsed %d nics", nets.size(), nicCount)); + } + } + + /** + * get all the stuff from parent node + * + * @param nic the object to carry through the system + * @param parentNode the xml container node for nic data + */ + private void fillNicPrerequisites(NetworkPrerequisiteTO nic, Node parentNode) { + String addressOnParentStr = getChildNodeValue(parentNode, "AddressOnParent"); + String automaticAllocationStr = getChildNodeValue(parentNode, "AutomaticAllocation"); + String description = getChildNodeValue(parentNode, "Description"); + String elementName = getChildNodeValue(parentNode, "ElementName"); + String instanceIdStr = getChildNodeValue(parentNode, "InstanceID"); + String resourceSubType = getChildNodeValue(parentNode, "ResourceSubType"); + String resourceType = getChildNodeValue(parentNode, "ResourceType"); + + try { + int addressOnParent = Integer.parseInt(addressOnParentStr); + nic.setAddressOnParent(addressOnParent); + } catch (NumberFormatException e) { + s_logger.warn("Encountered element of type \"AddressOnParent\", that could not be parse to an integer number: " + addressOnParentStr); + } + + boolean automaticAllocation = StringUtils.isNotBlank(automaticAllocationStr) && Boolean.parseBoolean(automaticAllocationStr); + nic.setAutomaticAllocation(automaticAllocation); + nic.setNicDescription(description); + nic.setElementName(elementName); + + try { + int instanceId = Integer.parseInt(instanceIdStr); + nic.setInstanceID(instanceId); + } catch (NumberFormatException e) { + s_logger.warn("Encountered element of type \"InstanceID\", that could not be parse to an integer number: " + instanceIdStr); + } + + nic.setResourceSubType(resourceSubType); + nic.setResourceType(resourceType); + } + + private void checkForOnlyOneSystemNode(Document doc) throws InternalErrorException { + // get hardware VirtualSystem, for now we support only one of those + NodeList systemElements = doc.getElementsByTagName("VirtualSystem"); + if (systemElements.getLength() != 1) { + String msg = "found " + systemElements.getLength() + " system definitions in OVA, can only handle exactly one."; + s_logger.warn(msg); + throw new InternalErrorException(msg); + } + } + + private Map getNetworksFromDocumentTree(Document doc) { + NodeList networkElements = doc.getElementsByTagName("Network"); + Map nets = new HashMap<>(); + for (int i = 0; i < networkElements.getLength(); i++) { + + Element networkElement = (Element)networkElements.item(i); + String networkName = getNodeAttribute(networkElement,"ovf","name"); + + String description = getChildNodeValue(networkElement, "Description"); + + NetworkPrerequisiteTO network = new NetworkPrerequisiteTO(); + network.setName(networkName); + network.setNetworkDescription(description); + + nets.put(networkName,network); + } + if (s_logger.isTraceEnabled()) { + s_logger.trace(String.format("found %d networks in template", nets.size())); + } + return nets; + } + + private boolean hardwareItemContainsConfiguration(OVFVirtualHardwareItemTO item, String configurationId) { + if (StringUtils.isBlank(configurationId) || StringUtils.isBlank(item.getConfigurationIds())) { + return true; + } + String configurationIds = item.getConfigurationIds(); + if (StringUtils.isNotBlank(configurationIds)) { + String[] configurations = configurationIds.split(" "); + List confList = Arrays.asList(configurations); + return confList.contains(configurationId); + } + return false; + } + + /** + * Retrieve the virtual hardware section and its deployment options as configurations + */ + public OVFVirtualHardwareSectionTO getVirtualHardwareSectionFromDocument(Document doc) { + List configurations = getDeploymentOptionsFromDocumentTree(doc); + List items = getVirtualHardwareItemsFromDocumentTree(doc); + if (CollectionUtils.isNotEmpty(configurations)) { + for (OVFConfigurationTO configuration : configurations) { + List confItems = items.stream(). + filter(x -> StringUtils.isNotBlank(x.getConfigurationIds()) + && hardwareItemContainsConfiguration(x, configuration.getId())) + .collect(Collectors.toList()); + configuration.setHardwareItems(confItems); + } + } + List commonItems = null; + if (CollectionUtils.isNotEmpty(items)) { + commonItems = items.stream().filter(x -> StringUtils.isBlank(x.getConfigurationIds())).collect(Collectors.toList()); + } + return new OVFVirtualHardwareSectionTO(configurations, commonItems); + } + + private List getDeploymentOptionsFromDocumentTree(Document doc) { + List options = new ArrayList<>(); + if (doc == null) { + return options; + } + NodeList deploymentOptionSection = doc.getElementsByTagName("DeploymentOptionSection"); + if (deploymentOptionSection.getLength() == 0) { + return options; + } + Node hardwareSectionNode = deploymentOptionSection.item(0); + NodeList childNodes = hardwareSectionNode.getChildNodes(); + int index = 0; + for (int i = 0; i < childNodes.getLength(); i++) { + Node node = childNodes.item(i); + if (node != null && node.getNodeName().equals("Configuration")) { + Element configuration = (Element) node; + String configurationId = getNodeAttribute(configuration,"ovf","id"); + String description = getChildNodeValue(configuration, "Description"); + String label = getChildNodeValue(configuration, "Label"); + OVFConfigurationTO option = new OVFConfigurationTO(configurationId, label, description, index); + options.add(option); + index++; + } + } + return options; + } + + private List getVirtualHardwareItemsFromDocumentTree(Document doc) { + List items = new LinkedList<>(); + if (doc == null) { + return items; + } + NodeList hardwareSection = doc.getElementsByTagName("VirtualHardwareSection"); + if (hardwareSection.getLength() == 0) { + return items; + } + Node hardwareSectionNode = hardwareSection.item(0); + NodeList childNodes = hardwareSectionNode.getChildNodes(); + for (int i = 0; i < childNodes.getLength(); i++) { + Node node = childNodes.item(i); + if (node != null && node.getNodeName().equals("Item")) { + Element configuration = (Element) node; + String configurationIds = getNodeAttribute(configuration, "ovf", "configuration"); + String allocationUnits = getChildNodeValue(configuration, "AllocationUnits"); + String description = getChildNodeValue(configuration, "Description"); + String elementName = getChildNodeValue(configuration, "ElementName"); + String instanceID = getChildNodeValue(configuration, "InstanceID"); + String limit = getChildNodeValue(configuration, "Limit"); + String reservation = getChildNodeValue(configuration, "Reservation"); + String resourceType = getChildNodeValue(configuration, "ResourceType"); + String virtualQuantity = getChildNodeValue(configuration, "VirtualQuantity"); + String hostResource = getChildNodeValue(configuration, "HostResource"); + String addressOnParent = getChildNodeValue(configuration, "AddressOnParent"); + String parent = getChildNodeValue(configuration, "Parent"); + OVFVirtualHardwareItemTO item = new OVFVirtualHardwareItemTO(); + item.setConfigurationIds(configurationIds); + item.setAllocationUnits(allocationUnits); + item.setDescription(description); + item.setElementName(elementName); + item.setInstanceId(instanceID); + item.setLimit(getLongValueFromString(limit)); + item.setReservation(getLongValueFromString(reservation)); + Integer resType = getIntValueFromString(resourceType); + if (resType != null) { + item.setResourceType(OVFVirtualHardwareItemTO.getResourceTypeFromId(resType)); + } + item.setVirtualQuantity(getLongValueFromString(virtualQuantity)); + item.setHostResource(hostResource); + item.setAddressOnParent(addressOnParent); + item.setParent(parent); + items.add(item); + } + } + return items; + } + + private Long getLongValueFromString(String value) { + if (StringUtils.isNotBlank(value)) { + try { + return Long.parseLong(value); + } catch (NumberFormatException e) { + s_logger.debug("Could not parse the value: " + value + ", ignoring it"); + } + } + return null; + } + + private Integer getIntValueFromString(String value) { + if (StringUtils.isNotBlank(value)) { + try { + return Integer.parseInt(value); + } catch (NumberFormatException e) { + s_logger.debug("Could not parse the value: " + value + ", ignoring it"); + } + } + return null; + } + + protected byte[] compressOVFEula(String license) throws IOException { + CompressionUtil compressionUtil = new CompressionUtil(); + return compressionUtil.compressString(license); + } + + public List getEulaSectionsFromDocument(Document doc) { + List eulas = new LinkedList<>(); + if (doc == null) { + return eulas; + } + NodeList eulaSections = doc.getElementsByTagName("EulaSection"); + int eulaIndex = 0; + if (eulaSections.getLength() > 0) { + for (int index = 0; index < eulaSections.getLength(); index++) { + Node eulaNode = eulaSections.item(index); + NodeList eulaChildNodes = eulaNode.getChildNodes(); + String eulaInfo = null; + String eulaLicense = null; + for (int i = 0; i < eulaChildNodes.getLength(); i++) { + Node eulaItem = eulaChildNodes.item(i); + if (eulaItem.getNodeName().equalsIgnoreCase("Info")) { + eulaInfo = eulaItem.getTextContent(); + } else if (eulaItem.getNodeName().equalsIgnoreCase("License")) { + eulaLicense = eulaItem.getTextContent(); + } + } + byte[] compressedLicense = new byte[0]; + try { + compressedLicense = compressOVFEula(eulaLicense); + } catch (IOException e) { + s_logger.error("Could not compress the license for info " + eulaInfo); + continue; + } + OVFEulaSectionTO eula = new OVFEulaSectionTO(eulaInfo, compressedLicense, eulaIndex); + eulas.add(eula); + eulaIndex++; + } + } + + return eulas; + } + class OVFFile { // public String _href; @@ -417,7 +877,6 @@ public class OVFHelper { // public Long _capacity; - public String _capacityUnit; public String _diskId; public String _fileRef; public Long _populatedSize; diff --git a/api/src/main/java/com/cloud/agent/api/storage/OVFPropertyTO.java b/api/src/main/java/com/cloud/agent/api/storage/OVFPropertyTO.java index abf743ae713..24207e92821 100644 --- a/api/src/main/java/com/cloud/agent/api/storage/OVFPropertyTO.java +++ b/api/src/main/java/com/cloud/agent/api/storage/OVFPropertyTO.java @@ -30,7 +30,7 @@ import com.cloud.agent.api.LogLevel; * Choose "Remote HTTP and SSH Client Routes" to route only traffic destined for the management client(s), when they are on remote networks. * */ -public class OVFPropertyTO implements OVFProperty { +public class OVFPropertyTO { private String key; private String type; @@ -41,18 +41,14 @@ public class OVFPropertyTO implements OVFProperty { private String label; private String description; private Boolean password; + private int index; + private String category; public OVFPropertyTO() { } - public OVFPropertyTO(String key, String value, boolean password) { - this.key = key; - this.value = value; - this.password = password; - } - public OVFPropertyTO(String key, String type, String value, String qualifiers, boolean userConfigurable, - String label, String description, boolean password) { + String label, String description, boolean password, int index, String category) { this.key = key; this.type = type; this.value = value; @@ -61,9 +57,10 @@ public class OVFPropertyTO implements OVFProperty { this.label = label; this.description = description; this.password = password; + this.index = index; + this.category = category; } - @Override public Long getTemplateId() { return null; } @@ -131,4 +128,12 @@ public class OVFPropertyTO implements OVFProperty { public void setPassword(Boolean password) { this.password = password; } + + public String getCategory() { + return category; + } + + public int getIndex() { + return index; + } } diff --git a/api/src/main/java/com/cloud/agent/api/storage/OVFVirtualHardwareItemTO.java b/api/src/main/java/com/cloud/agent/api/storage/OVFVirtualHardwareItemTO.java new file mode 100644 index 00000000000..365ef7d37b5 --- /dev/null +++ b/api/src/main/java/com/cloud/agent/api/storage/OVFVirtualHardwareItemTO.java @@ -0,0 +1,365 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.api.storage; + +// From: https://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData.xsd +public class OVFVirtualHardwareItemTO { + + //From: https://schemas.dmtf.org/wbem/cim-html/2/CIM_ResourceAllocationSettingData.html + public enum HardwareResourceType { + Other("Other", 1), + ComputerSystem ("Computer System", 2), + Processor("Processor", 3), + Memory("Memory", 4), + IDEController("IDE Controller", 5), + ParallelSCSIHBA("Parallel SCSI HBA", 6), + FC_HBA("FC HBA", 7), + iSCSI_HBA("iSCSI HBA", 8), + IB_HCA("IB HCA", 9), + EthernetAdapter("Ethernet Adaptor", 10), + OtherNetworkAdapter("Other Network Adaptor", 11), + IO_Slot("I/O Slot", 12), + IO_Device("I/O Device", 13), + FloppyDrive("Floppy Drive", 14), + CD_Drive("CD Drive", 15), + DVD_Drive("DVD Drive", 16), + DiskDrive("Disk Drive", 17), + TapeDrive("Tape Drive", 18), + StorageExtent("Storage Extent", 19), + OtherStorageDevice("Other Storage Device", 20), + SerialPort("Serial Port", 21), + ParallelPort("Parallel Port", 22), + USBController("USB Controller", 23), + GraphicsController("Graphics Controller", 24), + IEEE_1394_Controller("IEEE 1394 Controller", 25), + PartitionableUnit("Partitionable Unit", 26), + BasePartitionableUnit("base Partitionable Unit", 27), + PowerSupply("Power", 28), + CoolingCapacity("Cooling Capacity", 29), + EthernetSwitchPort("Ethernet Switch Port", 30), + LogicalDisk("Logical Disk", 31), + StorageVolume("Storage Volume", 32), + EthernetConnection("Ethernet Connection", 33), + DMTF_reserved("DMTF Reserved", 35), + VendorReserved("Vendor Reserved", 32768); + + private String name; + private int id; + + HardwareResourceType(String name, int id) { + this.name = name; + this.id = id; + } + + public String getName() { + return name; + } + } + + public static HardwareResourceType getResourceTypeFromId(int id) { + if (id <= 33) { + for (HardwareResourceType type : HardwareResourceType.values()) { + if (type.id == id) { + return type; + } + } + } else if (id <= 32767) { + return HardwareResourceType.DMTF_reserved; + } + return HardwareResourceType.VendorReserved; + } + + public enum CustomerVisibility { + Unknown, PassedThrough, Virtualized, NotRepresented, DMTFReserved, VendorReserved; + } + + public enum MappingBehavior { + Unknown, NotSupported, Dedicated, SoftAffinity, HardAffinity, DMTFReserved, VendorReserved; + } + + private String address; + private String addressOnParent; + private String allocationUnits; + private boolean automaticAllocation; + private boolean automaticDeallocation; + private String caption; + private String changeableType; + private String componentSetting; + private String configurationName; + private String connection; + private CustomerVisibility customerVisibility; + private String description; + private String elementName; + private Long generation; + private String hostResource; + private String instanceId; + private Long limit; + private MappingBehavior mappingBehavior; + private String otherResourceType; + private String parent; + private String poolId; + private Long reservation; + private String resourceSubtype; + private HardwareResourceType resourceType; + private String soId; + private String soOrgId; + private Long virtualQuantity; + private String virtualQuantityUnits; + private int weight; + + private String configurationIds; + + public String getConfigurationIds() { + return configurationIds; + } + + public void setConfigurationIds(String configurationIds) { + this.configurationIds = configurationIds; + } + + public String getAddress() { + return address; + } + + public void setAddress(String address) { + this.address = address; + } + + public String getAddressOnParent() { + return addressOnParent; + } + + public void setAddressOnParent(String addressOnParent) { + this.addressOnParent = addressOnParent; + } + + public String getAllocationUnits() { + return allocationUnits; + } + + public void setAllocationUnits(String allocationUnits) { + this.allocationUnits = allocationUnits; + } + + public boolean isAutomaticAllocation() { + return automaticAllocation; + } + + public void setAutomaticAllocation(boolean automaticAllocation) { + this.automaticAllocation = automaticAllocation; + } + + public boolean isAutomaticDeallocation() { + return automaticDeallocation; + } + + public void setAutomaticDeallocation(boolean automaticDeallocation) { + this.automaticDeallocation = automaticDeallocation; + } + + public String getCaption() { + return caption; + } + + public void setCaption(String caption) { + this.caption = caption; + } + + public String getChangeableType() { + return changeableType; + } + + public void setChangeableType(String changeableType) { + this.changeableType = changeableType; + } + + public String getComponentSetting() { + return componentSetting; + } + + public void setComponentSetting(String componentSetting) { + this.componentSetting = componentSetting; + } + + public String getConfigurationName() { + return configurationName; + } + + public void setConfigurationName(String configurationName) { + this.configurationName = configurationName; + } + + public String getConnection() { + return connection; + } + + public void setConnection(String connection) { + this.connection = connection; + } + + public CustomerVisibility getCustomerVisibility() { + return customerVisibility; + } + + public void setCustomerVisibility(CustomerVisibility customerVisibility) { + this.customerVisibility = customerVisibility; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getElementName() { + return elementName; + } + + public void setElementName(String elementName) { + this.elementName = elementName; + } + + public Long getGeneration() { + return generation; + } + + public void setGeneration(Long generation) { + this.generation = generation; + } + + public String getHostResource() { + return hostResource; + } + + public void setHostResource(String hostResource) { + this.hostResource = hostResource; + } + + public String getInstanceId() { + return instanceId; + } + + public void setInstanceId(String instanceId) { + this.instanceId = instanceId; + } + + public Long getLimit() { + return limit; + } + + public void setLimit(Long limit) { + this.limit = limit; + } + + public MappingBehavior getMappingBehavior() { + return mappingBehavior; + } + + public void setMappingBehavior(MappingBehavior mappingBehavior) { + this.mappingBehavior = mappingBehavior; + } + + public String getOtherResourceType() { + return otherResourceType; + } + + public void setOtherResourceType(String otherResourceType) { + this.otherResourceType = otherResourceType; + } + + public String getParent() { + return parent; + } + + public void setParent(String parent) { + this.parent = parent; + } + + public String getPoolId() { + return poolId; + } + + public void setPoolId(String poolId) { + this.poolId = poolId; + } + + public Long getReservation() { + return reservation; + } + + public void setReservation(Long reservation) { + this.reservation = reservation; + } + + public String getResourceSubtype() { + return resourceSubtype; + } + + public void setResourceSubtype(String resourceSubtype) { + this.resourceSubtype = resourceSubtype; + } + + public HardwareResourceType getResourceType() { + return resourceType; + } + + public void setResourceType(HardwareResourceType resourceType) { + this.resourceType = resourceType; + } + + public String getSoId() { + return soId; + } + + public void setSoId(String soId) { + this.soId = soId; + } + + public String getSoOrgId() { + return soOrgId; + } + + public void setSoOrgId(String soOrgId) { + this.soOrgId = soOrgId; + } + + public Long getVirtualQuantity() { + return virtualQuantity; + } + + public void setVirtualQuantity(Long virtualQuantity) { + this.virtualQuantity = virtualQuantity; + } + + public String getVirtualQuantityUnits() { + return virtualQuantityUnits; + } + + public void setVirtualQuantityUnits(String virtualQuantityUnits) { + this.virtualQuantityUnits = virtualQuantityUnits; + } + + public int getWeight() { + return weight; + } + + public void setWeight(int weight) { + this.weight = weight; + } +} \ No newline at end of file diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/TemplateOVFPropertiesDao.java b/api/src/main/java/com/cloud/agent/api/storage/OVFVirtualHardwareSectionTO.java similarity index 54% rename from engine/schema/src/main/java/com/cloud/storage/dao/TemplateOVFPropertiesDao.java rename to api/src/main/java/com/cloud/agent/api/storage/OVFVirtualHardwareSectionTO.java index eb78f2023ac..9a3ae1244eb 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/TemplateOVFPropertiesDao.java +++ b/api/src/main/java/com/cloud/agent/api/storage/OVFVirtualHardwareSectionTO.java @@ -1,3 +1,4 @@ +// // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information @@ -14,18 +15,29 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. - -package com.cloud.storage.dao; - -import com.cloud.storage.TemplateOVFPropertyVO; -import com.cloud.utils.db.GenericDao; +// +package com.cloud.agent.api.storage; import java.util.List; -public interface TemplateOVFPropertiesDao extends GenericDao { +public class OVFVirtualHardwareSectionTO { - boolean existsOption(long templateId, String key); - TemplateOVFPropertyVO findByTemplateAndKey(long templateId, String key); - void saveOptions(List opts); - List listByTemplateId(long templateId); -} + public OVFVirtualHardwareSectionTO() { + } + + private List configurations; + private List commonHardwareItems; + + public OVFVirtualHardwareSectionTO(List configurations, List commonHardwareItems) { + this.configurations = configurations; + this.commonHardwareItems = commonHardwareItems; + } + + public List getConfigurations() { + return configurations; + } + + public List getCommonHardwareItems() { + return commonHardwareItems; + } +} \ No newline at end of file diff --git a/api/src/main/java/com/cloud/agent/api/to/DatadiskTO.java b/api/src/main/java/com/cloud/agent/api/to/DatadiskTO.java index 1d3f91e25db..31a02b0bec3 100644 --- a/api/src/main/java/com/cloud/agent/api/to/DatadiskTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/DatadiskTO.java @@ -27,18 +27,14 @@ public class DatadiskTO { private boolean isIso; private String diskController; private String diskControllerSubType; + private int diskNumber; + private String configuration; public DatadiskTO() { } - public DatadiskTO(String path, long virtualSize, long fileSize, boolean bootable) { - this.path = path; - this.virtualSize = virtualSize; - this.fileSize = fileSize; - this.bootable = bootable; - } - - public DatadiskTO(String path, long virtualSize, long fileSize, String diskId, boolean isIso, boolean bootable, String controller, String controllerSubType) { + public DatadiskTO(String path, long virtualSize, long fileSize, String diskId, boolean isIso, boolean bootable, + String controller, String controllerSubType, int diskNumber, String configuration) { this.path = path; this.virtualSize = virtualSize; this.fileSize = fileSize; @@ -47,6 +43,8 @@ public class DatadiskTO { this.isIso = isIso; this.diskController = controller; this.diskControllerSubType = controllerSubType; + this.diskNumber = diskNumber; + this.configuration = configuration; } public String getPath() { @@ -105,4 +103,11 @@ public class DatadiskTO { this.diskControllerSubType = diskControllerSubType; } + public int getDiskNumber() { + return this.diskNumber; + } + + public String getConfiguration() { + return configuration; + } } \ No newline at end of file diff --git a/api/src/main/java/com/cloud/agent/api/to/DeployAsIsInfoTO.java b/api/src/main/java/com/cloud/agent/api/to/DeployAsIsInfoTO.java new file mode 100644 index 00000000000..c51c2632137 --- /dev/null +++ b/api/src/main/java/com/cloud/agent/api/to/DeployAsIsInfoTO.java @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.api.to; + +public class DeployAsIsInfoTO { + + private boolean deployAsIs; + private String templatePath; + private String deploymentConfiguration; + + public DeployAsIsInfoTO() { + } + + public boolean isDeployAsIs() { + return deployAsIs; + } + + public void setDeployAsIs(boolean deployAsIs) { + this.deployAsIs = deployAsIs; + } + + public String getTemplatePath() { + return templatePath; + } + + public void setTemplatePath(String templateInSecondaryPath) { + this.templatePath = templateInSecondaryPath; + } + + public String getDeploymentConfiguration() { + return deploymentConfiguration; + } + + public void setDeploymentConfiguration(String deploymentConfiguration) { + this.deploymentConfiguration = deploymentConfiguration; + } +} diff --git a/api/src/main/java/com/cloud/agent/api/to/VirtualMachineTO.java b/api/src/main/java/com/cloud/agent/api/to/VirtualMachineTO.java index dceacf0e65b..7dbc70d6629 100644 --- a/api/src/main/java/com/cloud/agent/api/to/VirtualMachineTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/VirtualMachineTO.java @@ -23,7 +23,6 @@ import java.util.HashMap; import com.cloud.agent.api.LogLevel; import com.cloud.agent.api.storage.OVFPropertyTO; import com.cloud.template.VirtualMachineTemplate.BootloaderType; -import com.cloud.utils.Pair; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachine.Type; @@ -82,7 +81,8 @@ public class VirtualMachineTO { Map guestOsDetails = new HashMap(); Map extraConfig = new HashMap<>(); @LogLevel(LogLevel.Log4jLevel.Off) - Pair> ovfProperties; + List ovfProperties; + DeployAsIsInfoTO deployAsIsInfo; public VirtualMachineTO(long id, String instanceName, VirtualMachine.Type type, int cpus, Integer speed, long minRam, long maxRam, BootloaderType bootloader, String os, boolean enableHA, boolean limitCpuUse, String vncPassword) { @@ -376,11 +376,11 @@ public class VirtualMachineTO { return extraConfig; } - public Pair> getOvfProperties() { + public List getOvfProperties() { return ovfProperties; } - public void setOvfProperties(Pair> ovfProperties) { + public void setOvfProperties(List ovfProperties) { this.ovfProperties = ovfProperties; } public String getBootType() { @@ -402,4 +402,12 @@ public class VirtualMachineTO { public void setEnterHardwareSetup(boolean enterHardwareSetup) { this.enterHardwareSetup = enterHardwareSetup; } + + public DeployAsIsInfoTO getDeployAsIsInfo() { + return deployAsIsInfo; + } + + public void setDeployAsIsInfo(DeployAsIsInfoTO deployAsIsInfo) { + this.deployAsIsInfo = deployAsIsInfo; + } } diff --git a/api/src/main/java/com/cloud/storage/ImageStore.java b/api/src/main/java/com/cloud/storage/ImageStore.java index c019b17421d..017f367c13e 100644 --- a/api/src/main/java/com/cloud/storage/ImageStore.java +++ b/api/src/main/java/com/cloud/storage/ImageStore.java @@ -21,6 +21,13 @@ import org.apache.cloudstack.api.InternalIdentity; public interface ImageStore extends Identity, InternalIdentity { + String ACS_PROPERTY_PREFIX = "ACS-property-"; + String REQUIRED_NETWORK_PREFIX = "ACS-network-"; + String DISK_DEFINITION_PREFIX = "ACS-disk-"; + String OVF_HARDWARE_CONFIGURATION_PREFIX = "ACS-configuration-"; + String OVF_HARDWARE_ITEM_PREFIX = "ACS-hardware-item-"; + String OVF_EULA_SECTION_PREFIX = "ACS-eula-"; + /** * @return name of the object store. */ diff --git a/api/src/main/java/com/cloud/template/VirtualMachineTemplate.java b/api/src/main/java/com/cloud/template/VirtualMachineTemplate.java index 5177e51d401..95d1ebf0b87 100644 --- a/api/src/main/java/com/cloud/template/VirtualMachineTemplate.java +++ b/api/src/main/java/com/cloud/template/VirtualMachineTemplate.java @@ -138,4 +138,6 @@ public interface VirtualMachineTemplate extends ControlledEntity, Identity, Inte void incrUpdatedCount(); Date getUpdated(); + + boolean isDeployAsIs(); } diff --git a/api/src/main/java/com/cloud/vm/NicProfile.java b/api/src/main/java/com/cloud/vm/NicProfile.java index 47021c85108..cd2215cf715 100644 --- a/api/src/main/java/com/cloud/vm/NicProfile.java +++ b/api/src/main/java/com/cloud/vm/NicProfile.java @@ -52,6 +52,8 @@ public class NicProfile implements InternalIdentity, Serializable { Integer networkRate; boolean isSecurityGroupEnabled; + Integer orderIndex; + // IPv4 String iPv4Address; String iPv4Netmask; @@ -381,6 +383,14 @@ public class NicProfile implements InternalIdentity, Serializable { this.requestedIPv6 = requestedIPv6; } + public Integer getOrderIndex() { + return orderIndex; + } + + public void setOrderIndex(Integer orderIndex) { + this.orderIndex = orderIndex; + } + // // OTHER METHODS // @@ -410,6 +420,8 @@ public class NicProfile implements InternalIdentity, Serializable { broadcastUri = null; isolationUri = null; + orderIndex = null; + } @Override diff --git a/api/src/main/java/com/cloud/vm/VmDetailConstants.java b/api/src/main/java/com/cloud/vm/VmDetailConstants.java index 3812aa21144..9991e1f35b4 100644 --- a/api/src/main/java/com/cloud/vm/VmDetailConstants.java +++ b/api/src/main/java/com/cloud/vm/VmDetailConstants.java @@ -63,4 +63,6 @@ public interface VmDetailConstants { String IP6_ADDRESS = "ip6Address"; String DISK = "disk"; String DISK_OFFERING = "diskOffering"; + + String DEPLOY_AS_IS_CONFIGURATION = "configurationId"; } diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 88f083b50e6..4751e312f92 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -185,6 +185,7 @@ public class ApiConstants { public static final String ICMP_TYPE = "icmptype"; public static final String ID = "id"; public static final String IDS = "ids"; + public static final String INDEX = "index"; public static final String PREVIOUS_ACL_RULE_ID = "previousaclruleid"; public static final String NEXT_ACL_RULE_ID = "nextaclruleid"; public static final String MOVE_ACL_CONSISTENCY_HASH = "aclconsistencyhash"; @@ -262,7 +263,7 @@ public class ApiConstants { public static final String OUTOFBANDMANAGEMENT_POWERSTATE = "outofbandmanagementpowerstate"; public static final String OUTOFBANDMANAGEMENT_ENABLED = "outofbandmanagementenabled"; public static final String OUTPUT = "output"; - public static final String OVF_PROPERTIES = "ovfproperties"; + public static final String PROPERTIES = "properties"; public static final String PARAMS = "params"; public static final String PARENT_ID = "parentid"; public static final String PARENT_DOMAIN_ID = "parentdomainid"; @@ -824,6 +825,10 @@ public class ApiConstants { public static final String BOOT_TYPE = "boottype"; public static final String BOOT_MODE = "bootmode"; public static final String BOOT_INTO_SETUP = "bootintosetup"; + public static final String DEPLOY_AS_IS = "deployasis"; + public static final String CROSS_ZONES = "crossZones"; + public static final String TEMPLATETYPE = "templatetype"; + public static final String SOURCETEMPLATEID = "sourcetemplateid"; public enum BootType { UEFI, BIOS; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportUnmanagedInstanceCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportUnmanagedInstanceCmd.java index 10321cc035d..5f924f213a0 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportUnmanagedInstanceCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportUnmanagedInstanceCmd.java @@ -204,6 +204,9 @@ public class ImportUnmanagedInstanceCmd extends BaseAsyncCmd { for (Map entry : (Collection>)nicNetworkList.values()) { String nic = entry.get(VmDetailConstants.NIC); String networkUuid = entry.get(VmDetailConstants.NETWORK); + if (LOGGER.isTraceEnabled()) { + LOGGER.trace(String.format("nic, '%s', goes on net, '%s'", nic, networkUuid)); + } if (Strings.isNullOrEmpty(nic) || Strings.isNullOrEmpty(networkUuid) || _entityMgr.findByUuid(Network.class, networkUuid) == null) { throw new InvalidParameterValueException(String.format("Network ID: %s for NIC ID: %s is invalid", networkUuid, nic)); } @@ -219,11 +222,14 @@ public class ImportUnmanagedInstanceCmd extends BaseAsyncCmd { for (Map entry : (Collection>)nicIpAddressList.values()) { String nic = entry.get(VmDetailConstants.NIC); String ipAddress = Strings.emptyToNull(entry.get(VmDetailConstants.IP4_ADDRESS)); + if (LOGGER.isTraceEnabled()) { + LOGGER.trace(String.format("nic, '%s', gets ip, '%s'", nic, ipAddress)); + } if (Strings.isNullOrEmpty(nic)) { throw new InvalidParameterValueException(String.format("NIC ID: '%s' is invalid for IP address mapping", nic)); } if (Strings.isNullOrEmpty(ipAddress)) { - throw new InvalidParameterValueException(String.format("IP address '%s' for NIC ID: %s is invalid", ipAddress, nic)); + throw new InvalidParameterValueException(String.format("Empty address for NIC ID: %s is invalid", nic)); } if (!Strings.isNullOrEmpty(ipAddress) && !ipAddress.equals("auto") && !NetUtils.isValidIp4(ipAddress)) { throw new InvalidParameterValueException(String.format("IP address '%s' for NIC ID: %s is invalid", ipAddress, nic)); @@ -239,12 +245,15 @@ public class ImportUnmanagedInstanceCmd extends BaseAsyncCmd { Map dataDiskToDiskOfferingMap = new HashMap<>(); if (MapUtils.isNotEmpty(dataDiskToDiskOfferingList)) { for (Map entry : (Collection>)dataDiskToDiskOfferingList.values()) { - String nic = entry.get(VmDetailConstants.DISK); + String disk = entry.get(VmDetailConstants.DISK); String offeringUuid = entry.get(VmDetailConstants.DISK_OFFERING); - if (Strings.isNullOrEmpty(nic) || Strings.isNullOrEmpty(offeringUuid) || _entityMgr.findByUuid(DiskOffering.class, offeringUuid) == null) { - throw new InvalidParameterValueException(String.format("Disk offering ID: %s for disk ID: %s is invalid", offeringUuid, nic)); + if (LOGGER.isTraceEnabled()) { + LOGGER.trace(String.format("disk, '%s', gets offering, '%s'", disk, offeringUuid)); } - dataDiskToDiskOfferingMap.put(nic, _entityMgr.findByUuid(DiskOffering.class, offeringUuid).getId()); + if (Strings.isNullOrEmpty(disk) || Strings.isNullOrEmpty(offeringUuid) || _entityMgr.findByUuid(DiskOffering.class, offeringUuid) == null) { + throw new InvalidParameterValueException(String.format("Disk offering ID: %s for disk ID: %s is invalid", offeringUuid, disk)); + } + dataDiskToDiskOfferingMap.put(disk, _entityMgr.findByUuid(DiskOffering.class, offeringUuid).getId()); } } return dataDiskToDiskOfferingMap; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListServiceOfferingsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListServiceOfferingsCmd.java index dcb1730cec6..91cac0937d4 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListServiceOfferingsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListServiceOfferingsCmd.java @@ -65,6 +65,24 @@ public class ListServiceOfferingsCmd extends BaseListDomainResourcesCmd { since = "4.13") private Long zoneId; + @Parameter(name = ApiConstants.CPU_NUMBER, + type = CommandType.INTEGER, + description = "the CPU number that listed offerings must support", + since = "4.15") + private Integer cpuNumber; + + @Parameter(name = ApiConstants.MEMORY, + type = CommandType.INTEGER, + description = "the RAM memory that listed offering must support", + since = "4.15") + private Integer memory; + + @Parameter(name = ApiConstants.CPU_SPEED, + type = CommandType.INTEGER, + description = "the CPU speed that listed offerings must support", + since = "4.15") + private Integer cpuSpeed; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -93,6 +111,18 @@ public class ListServiceOfferingsCmd extends BaseListDomainResourcesCmd { return zoneId; } + public Integer getCpuNumber() { + return cpuNumber; + } + + public Integer getMemory() { + return memory; + } + + public Integer getCpuSpeed() { + return cpuSpeed; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/template/ListTemplateOVFProperties.java b/api/src/main/java/org/apache/cloudstack/api/command/user/template/ListTemplateOVFProperties.java deleted file mode 100644 index 2a620c9abe5..00000000000 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/template/ListTemplateOVFProperties.java +++ /dev/null @@ -1,68 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -package org.apache.cloudstack.api.command.user.template; - -import com.cloud.exception.ConcurrentOperationException; -import com.cloud.exception.InsufficientCapacityException; -import com.cloud.exception.NetworkRuleConflictException; -import com.cloud.exception.ResourceAllocationException; -import com.cloud.exception.ResourceUnavailableException; -import org.apache.cloudstack.acl.RoleType; -import org.apache.cloudstack.api.APICommand; -import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.BaseCmd; -import org.apache.cloudstack.api.BaseListCmd; -import org.apache.cloudstack.api.Parameter; -import org.apache.cloudstack.api.ServerApiException; -import org.apache.cloudstack.api.response.ListResponse; -import org.apache.cloudstack.api.response.TemplateOVFPropertyResponse; -import org.apache.cloudstack.api.response.TemplateResponse; -import org.apache.cloudstack.context.CallContext; - -@APICommand(name = ListTemplateOVFProperties.APINAME, - description = "List template OVF properties if available.", - responseObject = TemplateOVFPropertyResponse.class, - authorized = {RoleType.Admin, RoleType.DomainAdmin, RoleType.ResourceAdmin, RoleType.User}) -public class ListTemplateOVFProperties extends BaseListCmd { - - public static final String APINAME = "listTemplateOvfProperties"; - - @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = TemplateResponse.class, - description = "the template ID", required = true) - private Long templateId; - - public Long getTemplateId() { - return templateId; - } - - @Override - public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { - ListResponse response = _queryService.listTemplateOVFProperties(this); - response.setResponseName(getCommandName()); - setResponseObject(response); - } - - @Override - public String getCommandName() { - return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX; - } - - @Override - public long getEntityOwnerId() { - return CallContext.current().getCallingAccount().getId(); - } -} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/template/ListTemplatesCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/template/ListTemplatesCmd.java index 5630d7fba42..157118285fc 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/template/ListTemplatesCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/template/ListTemplatesCmd.java @@ -16,8 +16,12 @@ // under the License. package org.apache.cloudstack.api.command.user.template; +import com.cloud.exception.InvalidParameterValueException; +import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; +import java.util.ArrayList; +import java.util.EnumSet; import java.util.List; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiCommandJobType; @@ -82,10 +86,35 @@ public class ListTemplatesCmd extends BaseListTaggedResourcesCmd implements User @Parameter(name = ApiConstants.PARENT_TEMPLATE_ID, type = CommandType.UUID, entityType = TemplateResponse.class, description = "list datadisk templates by parent template id", since = "4.4") private Long parentTemplateId; + @Parameter(name = ApiConstants.DETAILS, + type = CommandType.LIST, + collectionType = CommandType.STRING, + description = "comma separated list of template details requested, value can be a list of [ all, resource, min]") + private List viewDetails; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// + public EnumSet getDetails() throws InvalidParameterValueException { + EnumSet dv; + if (CollectionUtils.isEmpty(viewDetails)) { + dv = EnumSet.of(ApiConstants.DomainDetails.all); + } else { + try { + ArrayList dc = new ArrayList<>(); + for (String detail : viewDetails) { + dc.add(ApiConstants.DomainDetails.valueOf(detail)); + } + dv = EnumSet.copyOf(dc); + } catch (IllegalArgumentException e) { + throw new InvalidParameterValueException("The details parameter contains a non permitted value. The allowed values are " + + EnumSet.allOf(ApiConstants.DomainDetails.class)); + } + } + return dv; + } + public String getHypervisor() { return hypervisor; } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java index 7e0002dbe99..ca08c2a5ded 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java @@ -162,6 +162,13 @@ public class RegisterTemplateCmd extends BaseCmd implements UserCmd { description = "true if template should bypass Secondary Storage and be downloaded to Primary Storage on deployment") private Boolean directDownload; + @Parameter(name= ApiConstants.DEPLOY_AS_IS, + type = CommandType.BOOLEAN, + description = "VMware only: true if template should not strip and define disks and networks but leave those to the template definition", + since = "4.15" + ) + private Boolean deployAsIs; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -274,6 +281,10 @@ public class RegisterTemplateCmd extends BaseCmd implements UserCmd { return directDownload == null ? false : directDownload; } + public Boolean isDeployAsIs() { + return deployAsIs == null ? false : deployAsIs; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java index b4514b1c759..09455d45dff 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java @@ -52,6 +52,7 @@ import org.apache.cloudstack.api.response.TemplateResponse; import org.apache.cloudstack.api.response.UserVmResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.cloudstack.context.CallContext; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.log4j.Logger; @@ -72,6 +73,8 @@ import com.cloud.uservm.UserVm; import com.cloud.utils.net.Dhcp; import com.cloud.utils.net.NetUtils; import com.cloud.vm.VirtualMachine; +import com.cloud.vm.VmDetailConstants; +import com.google.common.base.Strings; @APICommand(name = "deployVirtualMachine", description = "Creates and automatically starts a virtual machine based on a service offering, disk offering, and template.", responseObject = UserVmResponse.class, responseView = ResponseView.Restricted, entityType = {VirtualMachine.class}, requestHasSensitiveInfo = false, responseHasSensitiveInfo = true) @@ -221,10 +224,16 @@ public class DeployVMCmd extends BaseAsyncCreateCustomIdCmd implements SecurityG @Parameter(name = ApiConstants.COPY_IMAGE_TAGS, type = CommandType.BOOLEAN, since = "4.13", description = "if true the image tags (if any) will be copied to the VM, default value is false") private Boolean copyImageTags; - @Parameter(name = ApiConstants.OVF_PROPERTIES, type = CommandType.MAP, since = "4.13", - description = "used to specify the OVF properties.") + @Parameter(name = ApiConstants.PROPERTIES, type = CommandType.MAP, since = "4.15", + description = "used to specify the vApp properties.") @LogLevel(LogLevel.Log4jLevel.Off) - private Map vmOvfProperties; + private Map vAppProperties; + + @Parameter(name = ApiConstants.NIC_NETWORK_LIST, type = CommandType.MAP, since = "4.15", + description = "VMware only: used to specify network mapping of a vApp VMware template registered \"as-is\"." + + " Example nicnetworklist[0].ip=Nic-101&nicnetworklist[0].network=uuid") + @LogLevel(LogLevel.Log4jLevel.Off) + private Map vAppNetworks; ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// @@ -311,11 +320,10 @@ public class DeployVMCmd extends BaseAsyncCreateCustomIdCmd implements SecurityG return null; } - - public Map getVmOVFProperties() { + public Map getVmProperties() { Map map = new HashMap<>(); - if (MapUtils.isNotEmpty(vmOvfProperties)) { - Collection parameterCollection = vmOvfProperties.values(); + if (MapUtils.isNotEmpty(vAppProperties)) { + Collection parameterCollection = vAppProperties.values(); Iterator iterator = parameterCollection.iterator(); while (iterator.hasNext()) { HashMap entry = (HashMap)iterator.next(); @@ -325,6 +333,32 @@ public class DeployVMCmd extends BaseAsyncCreateCustomIdCmd implements SecurityG return map; } + public Map getVmNetworkMap() { + Map map = new HashMap<>(); + if (MapUtils.isNotEmpty(vAppNetworks)) { + Collection parameterCollection = vAppNetworks.values(); + Iterator iterator = parameterCollection.iterator(); + while (iterator.hasNext()) { + HashMap entry = (HashMap) iterator.next(); + Integer nic; + try { + nic = Integer.valueOf(entry.get(VmDetailConstants.NIC)); + } catch (NumberFormatException nfe) { + nic = null; + } + String networkUuid = entry.get(VmDetailConstants.NETWORK); + if (s_logger.isTraceEnabled()) { + s_logger.trace(String.format("nic, '%s', goes on net, '%s'", nic, networkUuid)); + } + if (nic == null || Strings.isNullOrEmpty(networkUuid) || _entityMgr.findByUuid(Network.class, networkUuid) == null) { + throw new InvalidParameterValueException(String.format("Network ID: %s for NIC ID: %s is invalid", networkUuid, nic)); + } + map.put(nic, _entityMgr.findByUuid(Network.class, networkUuid).getId()); + } + } + return map; + } + public String getGroup() { return group; } @@ -374,6 +408,13 @@ public class DeployVMCmd extends BaseAsyncCreateCustomIdCmd implements SecurityG } public List getNetworkIds() { + if (MapUtils.isNotEmpty(vAppNetworks)) { + if (CollectionUtils.isNotEmpty(networkIds) || ipAddress != null || getIp6Address() != null || MapUtils.isNotEmpty(ipToNetworkList)) { + throw new InvalidParameterValueException(String.format("%s can't be specified along with %s, %s, %s", ApiConstants.NIC_NETWORK_LIST, ApiConstants.NETWORK_IDS, ApiConstants.IP_ADDRESS, ApiConstants.IP_NETWORK_LIST)); + } else { + return new ArrayList<>(); + } + } if (ipToNetworkList != null && !ipToNetworkList.isEmpty()) { if ((networkIds != null && !networkIds.isEmpty()) || ipAddress != null || getIp6Address() != null) { throw new InvalidParameterValueException("ipToNetworkMap can't be specified along with networkIds or ipAddress"); diff --git a/api/src/main/java/org/apache/cloudstack/api/net/NetworkPrerequisiteTO.java b/api/src/main/java/org/apache/cloudstack/api/net/NetworkPrerequisiteTO.java new file mode 100644 index 00000000000..e39849685de --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/net/NetworkPrerequisiteTO.java @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.net; + +/** + * container for the network prerequisites as found in the appliance template + * + * for OVA: + * {code} + * + * Management Network Interface + * + * {code} + * {code} + * + * 7 + * true + * Management0-0 + * E1000 Ethernet adapter on "Management Network" + * Network adapter 1 + * 6 + * E1000 + * 10 + * + * {code} + */ +public class NetworkPrerequisiteTO { + String name; + String networkDescription; + + int addressOnParent; + boolean automaticAllocation; + String nicDescription; + String elementName; + int InstanceID; + String resourceSubType; + String resourceType; + + public int getAddressOnParent() { + return addressOnParent; + } + + public void setAddressOnParent(int addressOnParent) { + this.addressOnParent = addressOnParent; + } + + public boolean isAutomaticAllocation() { + return automaticAllocation; + } + + public void setAutomaticAllocation(boolean automaticAllocation) { + this.automaticAllocation = automaticAllocation; + } + + public String getNicDescription() { + return nicDescription; + } + + public void setNicDescription(String nicDescription) { + this.nicDescription = nicDescription; + } + + public String getElementName() { + return elementName; + } + + public void setElementName(String elementName) { + this.elementName = elementName; + } + + public int getInstanceID() { + return InstanceID; + } + + public void setInstanceID(int instanceID) { + InstanceID = instanceID; + } + + public String getResourceSubType() { + return resourceSubType; + } + + public void setResourceSubType(String resourceSubType) { + this.resourceSubType = resourceSubType; + } + + public String getResourceType() { + return resourceType; + } + + public void setResourceType(String resourceType) { + this.resourceType = resourceType; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getNetworkDescription() { + return networkDescription; + } + + public void setNetworkDescription(String networkDescription) { + this.networkDescription = networkDescription; + } +} \ No newline at end of file diff --git a/api/src/main/java/org/apache/cloudstack/api/response/TemplateOVFPropertyResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/TemplateOVFPropertyResponse.java index 83455a3fe6e..ebe0d1cfd0d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/TemplateOVFPropertyResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/TemplateOVFPropertyResponse.java @@ -16,14 +16,14 @@ // under the License. package org.apache.cloudstack.api.response; -import com.cloud.agent.api.storage.OVFProperty; import com.cloud.serializer.Param; import com.google.gson.annotations.SerializedName; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseResponse; -import org.apache.cloudstack.api.EntityReference; -@EntityReference(value = OVFProperty.class) +/** + * the placeholder of parameters to fill for deployment + */ public class TemplateOVFPropertyResponse extends BaseResponse { @SerializedName(ApiConstants.KEY) @@ -58,6 +58,27 @@ public class TemplateOVFPropertyResponse extends BaseResponse { @Param(description = "the ovf property label") private String description; + @SerializedName(ApiConstants.INDEX) + @Param(description = "the ovf property index") + private Integer index; + + @SerializedName(ApiConstants.CATEGORY) + @Param(description = "the ovf property category") + private String category; + + @Override + public boolean equals(Object other) { + if (!(other instanceof TemplateOVFPropertyResponse)) { + return false; + } + return key != null && key.equals(((TemplateOVFPropertyResponse)other).key); + } + + @Override + public int hashCode() { + return key.hashCode(); + } + public String getKey() { return key; } @@ -121,4 +142,20 @@ public class TemplateOVFPropertyResponse extends BaseResponse { public void setPassword(Boolean password) { this.password = password; } + + public Integer getIndex() { + return index; + } + + public void setIndex(Integer index) { + this.index = index; + } + + public String getCategory() { + return category; + } + + public void setCategory(String category) { + this.category = category; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/TemplateResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/TemplateResponse.java index 094fe2aa566..457d29c7ab1 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/TemplateResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/TemplateResponse.java @@ -17,6 +17,7 @@ package org.apache.cloudstack.api.response; import java.util.Date; +import java.util.HashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -55,7 +56,7 @@ public class TemplateResponse extends BaseResponseWithTagInformation implements @Param(description = "the date this template was created") private Date created; - @SerializedName("removed") + @SerializedName(ApiConstants.REMOVED) @Param(description = "the date this template was removed") private Date removed; @@ -80,7 +81,7 @@ public class TemplateResponse extends BaseResponseWithTagInformation implements @Param(description = "true if this template is a featured template, false otherwise") private boolean featured; - @SerializedName("crossZones") + @SerializedName(ApiConstants.CROSS_ZONES) @Param(description = "true if the template is managed across all Zones, false otherwise") private boolean crossZones; @@ -122,7 +123,7 @@ public class TemplateResponse extends BaseResponseWithTagInformation implements @Param(description = "the physical size of the template") private Long physicalSize; - @SerializedName("templatetype") + @SerializedName(ApiConstants.TEMPLATETYPE) @Param(description = "the type of the template") private String templateType; @@ -146,7 +147,7 @@ public class TemplateResponse extends BaseResponseWithTagInformation implements @Param(description = "checksum of the template") private String checksum; - @SerializedName("sourcetemplateid") + @SerializedName(ApiConstants.SOURCETEMPLATEID) @Param(description = "the template ID of the parent template if present") private String sourcetemplateId; @@ -154,7 +155,7 @@ public class TemplateResponse extends BaseResponseWithTagInformation implements @Param(description = "the ID of the secondary storage host for the template") private String hostId; - @SerializedName("hostname") + @SerializedName(ApiConstants.HOST_NAME) @Param(description = "the name of the secondary storage host for the template") private String hostName; @@ -172,7 +173,7 @@ public class TemplateResponse extends BaseResponseWithTagInformation implements @SerializedName(ApiConstants.DETAILS) @Param(description = "additional key/value details tied with template") - private Map details; + private Map details; @SerializedName(ApiConstants.DOWNLOAD_DETAILS) @Param(description = "Lists the download progress of a template across all secondary storages") @@ -194,12 +195,18 @@ public class TemplateResponse extends BaseResponseWithTagInformation implements @Param(description = "KVM Only: true if template is directly downloaded to Primary Storage bypassing Secondary Storage") private Boolean directDownload; + @SerializedName(ApiConstants.DEPLOY_AS_IS) + @Param(description = "VMware only: true if template is deployed without orchestrating disks and networks but \"as-is\" defined in the template.") + private Boolean deployAsIs; + @SerializedName("parenttemplateid") @Param(description = "if Datadisk template, then id of the root disk template this template belongs to") + @Deprecated(since = "4.15") private String parentTemplateId; @SerializedName("childtemplates") @Param(description = "if root disk template, then ids of the datas disk templates this template owns") + @Deprecated(since = "4.15") private Set childTemplates; @SerializedName(ApiConstants.REQUIRES_HVM) @@ -360,14 +367,21 @@ public class TemplateResponse extends BaseResponseWithTagInformation implements this.projectName = projectName; } - public Map getDetails() { + public Map getDetails() { return this.details; } - public void setDetails(Map details) { + public void setDetails(Map details) { this.details = details; } + public void addDetail(String key, String value) { + if (this.details == null) { + setDetails(new HashMap<>()); + } + this.details.put(key,value); + } + public void setTags(Set tags) { this.tags = tags; } @@ -396,6 +410,10 @@ public class TemplateResponse extends BaseResponseWithTagInformation implements return directDownload; } + public void setDeployAsIs(Boolean deployAsIs) { + this.deployAsIs = deployAsIs; + } + public void setParentTemplateId(String parentTemplateId) { this.parentTemplateId = parentTemplateId; } diff --git a/api/src/main/java/org/apache/cloudstack/query/QueryService.java b/api/src/main/java/org/apache/cloudstack/query/QueryService.java index 0a400ed9ae3..717af5df8e9 100644 --- a/api/src/main/java/org/apache/cloudstack/query/QueryService.java +++ b/api/src/main/java/org/apache/cloudstack/query/QueryService.java @@ -44,7 +44,6 @@ import org.apache.cloudstack.api.command.user.project.ListProjectsCmd; import org.apache.cloudstack.api.command.user.resource.ListDetailOptionsCmd; import org.apache.cloudstack.api.command.user.securitygroup.ListSecurityGroupsCmd; import org.apache.cloudstack.api.command.user.tag.ListTagsCmd; -import org.apache.cloudstack.api.command.user.template.ListTemplateOVFProperties; import org.apache.cloudstack.api.command.user.template.ListTemplatesCmd; import org.apache.cloudstack.api.command.user.vm.ListVMsCmd; import org.apache.cloudstack.api.command.user.vmgroup.ListVMGroupsCmd; @@ -74,7 +73,6 @@ import org.apache.cloudstack.api.response.SecurityGroupResponse; import org.apache.cloudstack.api.response.ServiceOfferingResponse; import org.apache.cloudstack.api.response.StoragePoolResponse; import org.apache.cloudstack.api.response.StorageTagResponse; -import org.apache.cloudstack.api.response.TemplateOVFPropertyResponse; import org.apache.cloudstack.api.response.TemplateResponse; import org.apache.cloudstack.api.response.UserResponse; import org.apache.cloudstack.api.response.UserVmResponse; @@ -173,7 +171,5 @@ public interface QueryService { ListResponse listManagementServers(ListMgmtsCmd cmd); - ListResponse listTemplateOVFProperties(ListTemplateOVFProperties cmd); - List listRouterHealthChecks(GetRouterHealthCheckResultsCmd cmd); } diff --git a/api/src/test/java/com/cloud/agent/api/storage/OVFHelperTest.java b/api/src/test/java/com/cloud/agent/api/storage/OVFHelperTest.java index 8aa9852fb25..c4f71052496 100644 --- a/api/src/test/java/com/cloud/agent/api/storage/OVFHelperTest.java +++ b/api/src/test/java/com/cloud/agent/api/storage/OVFHelperTest.java @@ -40,16 +40,689 @@ public class OVFHelperTest { "" + ""; + private String ovfFileDeploymentOptionsSection = + "\n" + + " Deployment Configuration information\n" + + " \n" + + " \n" + + " Use this option to deploy an ASAv with a maximum throughput of 100 Mbps (uses 1 vCPU and 2 GB of memory).\n" + + " \n" + + " \n" + + " \n" + + " Use this option to deploy an ASAv with a maximum throughput of 1 Gbps (uses 1 vCPU and 2 GB of memory).\n" + + " \n" + + " \n" + + " \n" + + " Use this option to deploy an ASAv with a maximum throughput of 2 Gbps (uses 4 vCPUs and 8 GB of memory).\n" + + " \n" + + " "; + + private String ovfFileVirtualHardwareSection = + "\n" + + " Virtual hardware requirements\n" + + " \n" + + " Virtual Hardware Family\n" + + " 0\n" + + " ASAv\n" + + " vmx-08,vmx-09\n" + + " \n" + + " \n" + + " hertz * 10^6\n" + + " Number of Virtual CPUs\n" + + " 1 virtual CPU(s)\n" + + " 1\n" + + " 5000\n" + + " 1000\n" + + " 3\n" + + " 1\n" + + " \n" + + " \n" + + " hertz * 10^6\n" + + " Number of Virtual CPUs\n" + + " 4 virtual CPU(s)\n" + + " 1\n" + + " 20000\n" + + " 1000\n" + + " 3\n" + + " 4\n" + + " \n" + + " \n" + + " byte * 2^20\n" + + " Memory Size\n" + + " 2048MB of memory\n" + + " 2\n" + + " 2048\n" + + " 2048\n" + + " 4\n" + + " 2048\n" + + " \n" + + " \n" + + " byte * 2^20\n" + + " Memory Size\n" + + " 8192MB of memory\n" + + " 2\n" + + " 8192\n" + + " 8192\n" + + " 4\n" + + " 8192\n" + + " \n" + + " \n" + + " 0\n" + + " SCSI Controller\n" + + " SCSI controller 0\n" + + " 3\n" + + " lsilogic\n" + + " 6\n" + + " \n" + + " \n" + + " 0\n" + + " IDE Controller\n" + + " IDE 0\n" + + " 4\n" + + " 5\n" + + " \n" + + " \n" + + " 0\n" + + " true\n" + + " CD/DVD Drive\n" + + " 5\n" + + " 4\n" + + " 15\n" + + " \n" + + " \n" + + " 1\n" + + " true\n" + + " CD/DVD Drive\n" + + " ovf:/file/file3\n" + + " 18\n" + + " 4\n" + + " 15\n" + + " \n" + + " \n" + + " 7\n" + + " true\n" + + " Management0-0\n" + + " E1000 Ethernet adapter on \"Management Network\"\n" + + " Network adapter 1\n" + + " 6\n" + + " E1000\n" + + " 10\n" + + " \n" + + " \n" + + " 0\n" + + " Hard disk 1\n" + + " ovf:/disk/vmdisk1\n" + + " 7\n" + + " 3\n" + + " 17\n" + + " \n" + + " \n" + + " 1\n" + + " Hard disk 2\n" + + " ovf:/disk/vmdisk2\n" + + " 8\n" + + " 3\n" + + " 17\n" + + " \n" + + " \n" + + " 8\n" + + " true\n" + + " GigabitEthernet0-0\n" + + " General purpose E1000 Ethernet adapter\n" + + " Network adapter 2\n" + + " 9\n" + + " E1000\n" + + " 10\n" + + " \n" + + " \n" + + " 9\n" + + " true\n" + + " GigabitEthernet0-1\n" + + " General purpose E1000 Ethernet adapter\n" + + " Network adapter 3\n" + + " 10\n" + + " E1000\n" + + " 10\n" + + " \n" + + " \n" + + " 10\n" + + " true\n" + + " GigabitEthernet0-2\n" + + " General purpose E1000 Ethernet adapter\n" + + " Network adapter 4\n" + + " 11\n" + + " E1000\n" + + " 10\n" + + " \n" + + " \n" + + " 11\n" + + " true\n" + + " GigabitEthernet0-3\n" + + " General purpose E1000 Ethernet adapter\n" + + " Network adapter 5\n" + + " 12\n" + + " E1000\n" + + " 10\n" + + " \n" + + " \n" + + " 12\n" + + " true\n" + + " GigabitEthernet0-4\n" + + " General purpose E1000 Ethernet adapter\n" + + " Network adapter 6\n" + + " 13\n" + + " E1000\n" + + " 10\n" + + " \n" + + " \n" + + " 13\n" + + " true\n" + + " GigabitEthernet0-5\n" + + " General purpose E1000 Ethernet adapter\n" + + " Network adapter 7\n" + + " 14\n" + + " E1000\n" + + " 10\n" + + " \n" + + " \n" + + " 14\n" + + " true\n" + + " GigabitEthernet0-6\n" + + " General purpose E1000 Ethernet adapter\n" + + " Network adapter 8\n" + + " 15\n" + + " E1000\n" + + " 10\n" + + " \n" + + " \n" + + " 15\n" + + " true\n" + + " GigabitEthernet0-7\n" + + " General purpose E1000 Ethernet adapter\n" + + " Network adapter 9\n" + + " 16\n" + + " E1000\n" + + " 10\n" + + " \n" + + " \n" + + " 16\n" + + " true\n" + + " GigabitEthernet0-8\n" + + " Default HA failover E1000 Ethernet adapter, or additional standalone general purpose adapter\n" + + " Network adapter 10\n" + + " 17\n" + + " E1000\n" + + " 10\n" + + " \n" + + " \n" + + " "; + + private String eulaSections = + "\n" + + "\n" + + " end-user license agreement\n" + + " END USER LICENSE AGREEMENT\n" + + "\n" + + "IMPORTANT: PLEASE READ THIS END USER LICENSE AGREEMENT CAREFULLY. IT IS VERY IMPORTANT THAT YOU CHECK THAT YOU ARE PURCHASING CISCO SOFTWARE OR EQUIPMENT FROM AN APPROVED SOURCE AND THAT YOU, OR THE ENTITY YOU REPRESENT (COLLECTIVELY, THE \"CUSTOMER\") HAVE BEEN REGISTERED AS THE END USER FOR THE PURPOSES OF THIS CISCO END USER LICENSE AGREEMENT. IF YOU ARE NOT REGISTERED AS THE END USER YOU HAVE NO LICENSE TO USE THE SOFTWARE AND THE LIMITED WARRANTY IN THIS END USER LICENSE AGREEMENT DOES NOT APPLY. ASSUMING YOU HAVE PURCHASED FROM AN APPROVED SOURCE, DOWNLOADING, INSTALLING OR USING CISCO OR CISCO-SUPPLIED SOFTWARE CONSTITUTES ACCEPTANCE OF THIS AGREEMENT.\n" + + "\n" + + "CISCO SYSTEMS, INC. OR ITS AFFILIATE LICENSING THE SOFTWARE (\"CISCO\") IS WILLING TO LICENSE THIS SOFTWARE TO YOU ONLY UPON THE CONDITION THAT YOU PURCHASED THE SOFTWARE FROM AN APPROVED SOURCE AND THAT YOU ACCEPT ALL OF THE TERMS CONTAINED IN THIS END USER LICENSE AGREEMENT PLUS ANY ADDITIONAL LIMITATIONS ON THE LICENSE SET FORTH IN A SUPPLEMENTAL LICENSE AGREEMENT ACCOMPANYING THE PRODUCT, MADE AVAILABLE AT THE TIME OF YOUR ORDER, OR POSTED ON THE CISCO WEBSITE AT www.cisco.com/go/terms (COLLECTIVELY THE \"AGREEMENT\"). TO THE EXTENT OF ANY CONFLICT BETWEEN THE TERMS OF THIS END USER LICENSE AGREEMENT AND ANY SUPPLEMENTAL LICENSE AGREEMENT, THE SUPPLEMENTAL LICENSE AGREEMENT SHALL APPLY. BY DOWNLOADING, INSTALLING, OR USING THE SOFTWARE, YOU ARE REPRESENTING THAT YOU PURCHASED THE SOFTWARE FROM AN APPROVED SOURCE AND BINDING YOURSELF TO THE AGREEMENT. IF YOU DO " + + "NOT AGREE TO ALL OF THE TERMS OF THE AGREEMENT, THEN CISCO IS UNWILLING TO LICENSE THE SOFTWARE TO YOU AND (A) YOU MAY NOT DOWNLOAD, INSTALL OR USE THE SOFTWARE, AND (B) YOU MAY RETURN THE SOFTWARE (INCLUDING ANY UNOPENED CD PACKAGE AND ANY WRITTEN MATERIALS) FOR A FULL REFUND, OR, IF THE SOFTWARE AND WRITTEN MATERIALS ARE SUPPLIED AS PART OF ANOTHER PRODUCT, YOU MAY RETURN THE ENTIRE PRODUCT FOR A FULL REFUND. YOUR RIGHT TO RETURN AND REFUND EXPIRES 30 DAYS AFTER PURCHASE FROM AN APPROVED SOURCE, AND APPLIES ONLY IF YOU ARE THE ORIGINAL AND REGISTERED END USER PURCHASER. FOR THE PURPOSES OF THIS END USER LICENSE AGREEMENT, AN \"APPROVED SOURCE\" MEANS (A) CISCO; OR (B) A DISTRIBUTOR OR SYSTEMS INTEGRATOR AUTHORIZED BY CISCO TO DISTRIBUTE / SELL CISCO EQUIPMENT, SOFTWARE AND SERVICES WITHIN YOUR TERRITORY TO END " + + "USERS; OR (C) A RESELLER AUTHORIZED BY ANY SUCH DISTRIBUTOR OR SYSTEMS INTEGRATOR IN ACCORDANCE WITH THE TERMS OF THE DISTRIBUTOR'S AGREEMENT WITH CISCO TO DISTRIBUTE / SELL THE CISCO EQUIPMENT, SOFTWARE AND SERVICES WITHIN YOUR TERRITORY TO END USERS.\n" + + "\n" + + "THE FOLLOWING TERMS OF THE AGREEMENT GOVERN CUSTOMER'S USE OF THE SOFTWARE (DEFINED BELOW), EXCEPT TO THE EXTENT: (A) THERE IS A SEPARATE SIGNED CONTRACT BETWEEN CUSTOMER AND CISCO GOVERNING CUSTOMER'S USE OF THE SOFTWARE, OR (B) THE SOFTWARE INCLUDES A SEPARATE \"CLICK-ACCEPT\" LICENSE AGREEMENT OR THIRD PARTY LICENSE AGREEMENT AS PART OF THE INSTALLATION OR DOWNLOAD PROCESS GOVERNING CUSTOMER'S USE OF THE SOFTWARE. TO THE EXTENT OF A CONFLICT BETWEEN THE PROVISIONS OF THE FOREGOING DOCUMENTS, THE ORDER OF PRECEDENCE SHALL BE (1)THE SIGNED CONTRACT, (2) THE CLICK-ACCEPT AGREEMENT OR THIRD PARTY LICENSE AGREEMENT, AND (3) THE AGREEMENT. FOR PURPOSES OF THE AGREEMENT, \"SOFTWARE\" SHALL MEAN COMPUTER PROGRAMS, INCLUDING FIRMWARE AND COMPUTER PROGRAMS EMBEDDED IN CISCO EQUIPMENT, AS PROVIDED TO CUSTOMER BY AN APPROVED SOURCE, AND ANY UPGRADES, UPDATES, BUG FIXES " + + "OR MODIFIED VERSIONS THERETO (COLLECTIVELY, \"UPGRADES\"), ANY OF THE SAME WHICH HAS BEEN RELICENSED UNDER THE CISCO SOFTWARE TRANSFER AND RE-LICENSING POLICY (AS MAY BE AMENDED BY CISCO FROM TIME TO TIME) OR BACKUP COPIES OF ANY OF THE FOREGOING.\n" + + "\n" + + "License. Conditioned upon compliance with the terms and conditions of the Agreement, Cisco grants to Customer a nonexclusive and nontransferable license to use for Customer's internal business purposes the Software and the Documentation for which Customer has paid the required license fees to an Approved Source. \"Documentation\" means written information (whether contained in user or technical manuals, training materials, specifications or otherwise) pertaining to the Software and made available by an Approved Source with the Software in any manner (including on CD-Rom, or on-line). In order to use the Software, Customer may be required to input a registration number or product authorization key and register Customer's copy of the Software online at Cisco's website to obtain the necessary license key or license file.\n" + + "\n" + + "Customer's license to use the Software shall be limited to, and Customer shall not use the Software in excess of, a single hardware chassis or card or such other limitations as are set forth in the applicable Supplemental License Agreement or in the applicable purchase order which has been accepted by an Approved Source and for which Customer has paid to an Approved Source the required license fee (the \"Purchase Order\").\n" + + "\n" + + "Unless otherwise expressly provided in the Documentation or any applicable Supplemental License Agreement, Customer shall use the Software solely as embedded in, for execution on, or (where the applicable Documentation permits installation on non-Cisco equipment) for communication with Cisco equipment owned or leased by Customer and used for Customer's internal business purposes. No other licenses are granted by implication, estoppel or otherwise.\n" + + "\n" + + "For evaluation or beta copies for which Cisco does not charge a license fee, the above requirement to pay license fees does not apply.\n" + + "\n" + + "General Limitations. This is a license, not a transfer of title, to the Software and Documentation, and Cisco retains ownership of all copies of the Software and Documentation. Customer acknowledges that the Software and Documentation contain trade secrets of Cisco or its suppliers or licensors, including but not limited to the specific internal design and structure of individual programs and associated interface information. Except as otherwise expressly provided under the Agreement, Customer shall only use the Software in connection with the use of Cisco equipment purchased by the Customer from an Approved Source and Customer shall have no right, and Customer specifically agrees not to:\n" + + "\n" + + "(i) transfer, assign or sublicense its license rights to any other person or entity (other than in compliance with any Cisco relicensing/transfer policy then in force), or use the Software on Cisco equipment not purchased by the Customer from an Approved Source or on secondhand Cisco equipment, and Customer acknowledges that any attempted transfer, assignment, sublicense or use shall be void;\n" + + "\n" + + "(ii) make error corrections to or otherwise modify or adapt the Software or create derivative works based upon the Software, or permit third parties to do the same;\n" + + "\n" + + "(iii) reverse engineer or decompile, decrypt, disassemble or otherwise reduce the Software to human-readable form, except to the extent otherwise expressly permitted under applicable law notwithstanding this restriction or except to the extent that Cisco is legally required to permit such specific activity pursuant to any applicable open source license;\n" + + "\n" + + "(iv) publish any results of benchmark tests run on the Software;\n" + + "\n" + + "(v) use or permit the Software to be used to perform services for third parties, whether on a service bureau or time sharing basis or otherwise, without the express written authorization of Cisco; or\n" + + "\n" + + "(vi) disclose, provide, or otherwise make available trade secrets contained within the Software and Documentation in any form to any third party without the prior written consent of Cisco. Customer shall implement reasonable security measures to protect such trade secrets.\n" + + "\n" + + "To the extent required by applicable law, and at Customer's written request, Cisco shall provide Customer with the interface information needed to achieve interoperability between the Software and another independently created program, on payment of Cisco's applicable fee, if any. Customer shall observe strict obligations of confidentiality with respect to such information and shall use such information in compliance with any applicable terms and conditions upon which Cisco makes such information available.\n" + + "\n" + + "Software, Upgrades and Additional Copies. NOTWITHSTANDING ANY OTHER PROVISION OF THE AGREEMENT: (1) CUSTOMER HAS NO LICENSE OR RIGHT TO MAKE OR USE ANY ADDITIONAL COPIES OR UPGRADES UNLESS CUSTOMER, AT THE TIME OF MAKING OR ACQUIRING SUCH COPY OR UPGRADE, ALREADY HOLDS A VALID LICENSE TO THE ORIGINAL SOFTWARE AND HAS PAID THE APPLICABLE FEE TO AN APPROVED SOURCE FOR THE UPGRADE OR ADDITIONAL COPIES; (2) USE OF UPGRADES IS LIMITED TO CISCO EQUIPMENT SUPPLIED BY AN APPROVED SOURCE FOR WHICH CUSTOMER IS THE ORIGINAL END USER PURCHASER OR LESSEE OR OTHERWISE HOLDS A VALID LICENSE TO USE THE SOFTWARE WHICH IS BEING UPGRADED; AND (3) THE MAKING AND USE OF ADDITIONAL COPIES IS LIMITED TO NECESSARY BACKUP PURPOSES ONLY.\n" + + "\n" + + "Proprietary Notices. Customer agrees to maintain and reproduce all copyright, proprietary, and other notices on all copies, in any form, of the Software in the same form and manner that such copyright and other proprietary notices are included on the Software. Except as expressly authorized in the Agreement, Customer shall not make any copies or duplicates of any Software without the prior written permission of Cisco.\n" + + "\n" + + "Term and Termination. The Agreement and the license granted herein shall remain effective until terminated. Customer may terminate the Agreement and the license at any time by destroying all copies of Software and any Documentation. Customer's rights under the Agreement will terminate immediately without notice from Cisco if Customer fails to comply with any provision of the Agreement. Upon termination, Customer shall destroy all copies of Software and Documentation in its possession or control. All confidentiality obligations of Customer, all restrictions and limitations imposed on the Customer under the section titled \"General Limitations\" and all limitations of liability and disclaimers and restrictions of warranty shall survive termination of this Agreement. In addition, the provisions of the sections titled \"U.S. Government End User Purchasers\" and \"General Terms Applicable to the Limited Warranty Statement " + + "and End User License Agreement\" shall survive termination of the Agreement.\n" + + "\n" + + "Customer Records. Customer grants to Cisco and its independent accountants the right to examine Customer's books, records and accounts during Customer's normal business hours to verify compliance with this Agreement. In the event such audit discloses non-compliance with this Agreement, Customer shall promptly pay to Cisco the appropriate license fees, plus the reasonable cost of conducting the audit.\n" + + "\n" + + "Export, Re-Export, Transfer and Use Controls. The Software, Documentation and technology or direct products thereof (hereafter referred to as Software and Technology), supplied by Cisco under the Agreement are subject to export controls under the laws and regulations of the United States (\"U.S.\") and any other applicable countries' laws and regulations. Customer shall comply with such laws and regulations governing export, re-export, import, transfer and use of Cisco Software and Technology and will obtain all required U.S. and local authorizations, permits, or licenses. Cisco and Customer each agree to provide the other information, support documents, and assistance as may reasonably be required by the other in connection with securing authorizations or licenses. Information regarding compliance with export, re-export, transfer and use may be located at the following URL: " + + "www.cisco.com/web/about/doing_business/legal/global_export_trade/general_export/contract_compliance.html\n" + + "\n" + + "U.S. Government End User Purchasers. The Software and Documentation qualify as \"commercial items,\" as that term is defined at Federal Acquisition Regulation (\"FAR\") (48 C.F.R.) 2.101, consisting of \"commercial computer software\" and \"commercial computer software documentation\" as such terms are used in FAR 12.212. Consistent with FAR 12.212 and DoD FAR Supp. 227.7202-1 through 227.7202-4, and notwithstanding any other FAR or other contractual clause to the contrary in any agreement into which the Agreement may be incorporated, Customer may provide to Government end user or, if the Agreement is direct, Government end user will acquire, the Software and Documentation with only those rights set forth in the Agreement. Use of either the Software or Documentation or both constitutes agreement by the Government that the Software and Documentation are \"commercial computer software\" and \"commercial computer " + + "software documentation,\" and constitutes acceptance of the rights and restrictions herein.\n" + + "\n" + + "Identified Components; Additional Terms. The Software may contain or be delivered with one or more components, which may include third-party components, identified by Cisco in the Documentation, readme.txt file, third-party click-accept or elsewhere (e.g. on www.cisco.com) (the \"Identified Component(s)\") as being subject to different license agreement terms, disclaimers of warranties, limited warranties or other terms and conditions (collectively, \"Additional Terms\") than those set forth herein. You agree to the applicable Additional Terms for any such Identified Component(s).\n" + + "\n" + + "Limited Warranty\n" + + "\n" + + "Subject to the limitations and conditions set forth herein, Cisco warrants that commencing from the date of shipment to Customer (but in case of resale by an Approved Source other than Cisco, commencing not more than ninety (90) days after original shipment by Cisco), and continuing for a period of the longer of (a) ninety (90) days or (b) the warranty period (if any) expressly set forth as applicable specifically to software in the warranty card accompanying the product of which the Software is a part (the \"Product\") (if any): (a) the media on which the Software is furnished will be free of defects in materials and workmanship under normal use; and (b) the Software substantially conforms to the Documentation. The date of shipment of a Product by Cisco is set forth on the packaging material in which the Product is shipped. Except for the foregoing, the Software is provided \"AS IS\". This limited warranty extends only to the " + + "Software purchased from an Approved Source by a Customer who is the first registered end user. Customer's sole and exclusive remedy and the entire liability of Cisco and its suppliers under this limited warranty will be (i) replacement of defective media and/or (ii) at Cisco's option, repair, replacement, or refund of the purchase price of the Software, in both cases subject to the condition that any error or defect constituting a breach of this limited warranty is reported to the Approved Source supplying the Software to Customer, within the warranty period. Cisco or the Approved Source supplying the Software to Customer may, at its option, require return of the Software and/or Documentation as a condition to the remedy. In no event does Cisco warrant that the Software is error free or that Customer will be able to operate the Software without problems or interruptions. In addition, due to the continual development of new " + + "techniques for intruding upon and attacking networks, Cisco does not warrant that the Software or any equipment, system or network on which the Software is used will be free of vulnerability to intrusion or attack.\n" + + "\n" + + "Restrictions. This warranty does not apply if the Software, Product or any other equipment upon which the Software is authorized to be used (a) has been altered, except by Cisco or its authorized representative, (b) has not been installed, operated, repaired, or maintained in accordance with instructions supplied by Cisco, (c) has been subjected to abnormal physical or electrical stress, abnormal environmental conditions, misuse, negligence, or accident; or (d) is licensed for beta, evaluation, testing or demonstration purposes. The Software warranty also does not apply to (e) any temporary Software modules; (f) any Software not posted on Cisco's Software Center; (g) any Software that Cisco expressly provides on an \"AS IS\" basis on Cisco's Software Center; (h) any Software for which an Approved Source does not receive a license fee; and (i) Software supplied by any third party which is not an Approved Source.\n" + + "\n" + + "DISCLAIMER OF WARRANTY\n" + + "\n" + + "EXCEPT AS SPECIFIED IN THIS WARRANTY SECTION, ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS, AND WARRANTIES INCLUDING, WITHOUT LIMITATION, ANY IMPLIED WARRANTY OR CONDITION OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, SATISFACTORY QUALITY, NON-INTERFERENCE, ACCURACY OF INFORMATIONAL CONTENT, OR ARISING FROM A COURSE OF DEALING, LAW, USAGE, OR TRADE PRACTICE, ARE HEREBY EXCLUDED TO THE EXTENT ALLOWED BY APPLICABLE LAW AND ARE EXPRESSLY DISCLAIMED BY CISCO, ITS SUPPLIERS AND LICENSORS. TO THE EXTENT THAT ANY OF THE SAME CANNOT BE EXCLUDED, SUCH IMPLIED CONDITION, REPRESENTATION AND/OR WARRANTY IS LIMITED IN DURATION TO THE EXPRESS WARRANTY PERIOD REFERRED TO IN THE \"LIMITED WARRANTY\" SECTION ABOVE. BECAUSE SOME STATES OR JURISDICTIONS DO NOT ALLOW LIMITATIONS ON HOW LONG AN IMPLIED WARRANTY LASTS, THE ABOVE LIMITATION MAY NOT APPLY IN SUCH STATES. THIS WARRANTY GIVES CUSTOMER SPECIFIC LEGAL RIGHTS, " + + "AND CUSTOMER MAY ALSO HAVE OTHER RIGHTS WHICH VARY FROM JURISDICTION TO JURISDICTION. This disclaimer and exclusion shall apply even if the express warranty set forth above fails of its essential purpose.\n" + + "\n" + + "Disclaimer of Liabilities-Limitation of Liability. IF YOU ACQUIRED THE SOFTWARE IN THE UNITED STATES, LATIN AMERICA, CANADA, JAPAN OR THE CARIBBEAN, NOTWITHSTANDING ANYTHING ELSE IN THE AGREEMENT TO THE CONTRARY, ALL LIABILITY OF CISCO, ITS AFFILIATES, OFFICERS, DIRECTORS, EMPLOYEES, AGENTS, SUPPLIERS AND LICENSORS COLLECTIVELY, TO CUSTOMER, WHETHER IN CONTRACT, TORT (INCLUDING NEGLIGENCE), BREACH OF WARRANTY OR OTHERWISE, SHALL NOT EXCEED THE PRICE PAID BY CUSTOMER TO ANY APPROVED SOURCE FOR THE SOFTWARE THAT GAVE RISE TO THE CLAIM OR IF THE SOFTWARE IS PART OF ANOTHER PRODUCT, THE PRICE PAID FOR SUCH OTHER PRODUCT. THIS LIMITATION OF LIABILITY FOR SOFTWARE IS CUMULATIVE AND NOT PER INCIDENT (I.E. THE EXISTENCE OF TWO OR MORE CLAIMS WILL NOT ENLARGE THIS LIMIT).\n" + + "\n" + + "IF YOU ACQUIRED THE SOFTWARE IN EUROPE, THE MIDDLE EAST, AFRICA, ASIA OR OCEANIA, NOTWITHSTANDING ANYTHING ELSE IN THE AGREEMENT TO THE CONTRARY, ALL LIABILITY OF CISCO, ITS AFFILIATES, OFFICERS, DIRECTORS, EMPLOYEES, AGENTS, SUPPLIERS AND LICENSORS COLLECTIVELY, TO CUSTOMER, WHETHER IN CONTRACT, TORT (INCLUDING NEGLIGENCE), BREACH OF WARRANTY OR OTHERWISE, SHALL NOT EXCEED THE PRICE PAID BY CUSTOMER TO CISCO FOR THE SOFTWARE THAT GAVE RISE TO THE CLAIM OR IF THE SOFTWARE IS PART OF ANOTHER PRODUCT, THE PRICE PAID FOR SUCH OTHER PRODUCT. THIS LIMITATION OF LIABILITY FOR SOFTWARE IS CUMULATIVE AND NOT PER INCIDENT (I.E. THE EXISTENCE OF TWO OR MORE CLAIMS WILL NOT ENLARGE THIS LIMIT). NOTHING IN THE AGREEMENT SHALL LIMIT (I) THE LIABILITY OF CISCO, ITS AFFILIATES, OFFICERS, DIRECTORS, EMPLOYEES, AGENTS, SUPPLIERS AND LICENSORS TO CUSTOMER FOR PERSONAL INJURY OR DEATH CAUSED BY THEIR NEGLIGENCE, (II) CISCO'S LIABILITY FOR FRAUDULENT" + + " MISREPRESENTATION, OR (III) ANY LIABILITY OF CISCO WHICH CANNOT BE EXCLUDED UNDER APPLICABLE LAW.\n" + + "\n" + + "Disclaimer of Liabilities-Waiver of Consequential Damages and Other Losses. IF YOU ACQUIRED THE SOFTWARE IN THE UNITED STATES, LATIN AMERICA, THE CARIBBEAN OR CANADA, REGARDLESS OF WHETHER ANY REMEDY SET FORTH HEREIN FAILS OF ITS ESSENTIAL PURPOSE OR OTHERWISE, IN NO EVENT WILL CISCO OR ITS SUPPLIERS BE LIABLE FOR ANY LOST REVENUE, PROFIT, OR LOST OR DAMAGED DATA, BUSINESS INTERRUPTION, LOSS OF CAPITAL, OR FOR SPECIAL, INDIRECT, CONSEQUENTIAL, INCIDENTAL, OR PUNITIVE DAMAGES HOWEVER CAUSED AND REGARDLESS OF THE THEORY OF LIABILITY OR WHETHER ARISING OUT OF THE USE OF OR INABILITY TO USE SOFTWARE OR OTHERWISE AND EVEN IF CISCO OR ITS SUPPLIERS OR LICENSORS HAVE BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. BECAUSE SOME STATES OR JURISDICTIONS DO NOT ALLOW LIMITATION OR EXCLUSION OF CONSEQUENTIAL OR INCIDENTAL DAMAGES, THE ABOVE LIMITATION MAY NOT APPLY TO YOU.\n" + + "\n" + + "IF YOU ACQUIRED THE SOFTWARE IN JAPAN, EXCEPT FOR LIABILITY ARISING OUT OF OR IN CONNECTION WITH DEATH OR PERSONAL INJURY, FRAUDULENT MISREPRESENTATION, AND REGARDLESS OF WHETHER ANY REMEDY SET FORTH HEREIN FAILS OF ITS ESSENTIAL PURPOSE OR OTHERWISE, IN NO EVENT WILL CISCO, ITS AFFILIATES, OFFICERS, DIRECTORS, EMPLOYEES, AGENTS, SUPPLIERS AND LICENSORS BE LIABLE FOR ANY LOST REVENUE, PROFIT, OR LOST OR DAMAGED DATA, BUSINESS INTERRUPTION, LOSS OF CAPITAL, OR FOR SPECIAL, INDIRECT, CONSEQUENTIAL, INCIDENTAL, OR PUNITIVE DAMAGES HOWEVER CAUSED AND REGARDLESS OF THE THEORY OF LIABILITY OR WHETHER ARISING OUT OF THE USE OF OR INABILITY TO USE SOFTWARE OR OTHERWISE AND EVEN IF CISCO OR ANY APPROVED SOURCE OR THEIR SUPPLIERS OR LICENSORS HAVE BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.\n" + + "\n" + + "IF YOU ACQUIRED THE SOFTWARE IN EUROPE, THE MIDDLE EAST, AFRICA, ASIA OR OCEANIA, IN NO EVENT WILL CISCO, ITS AFFILIATES, OFFICERS, DIRECTORS, EMPLOYEES, AGENTS, SUPPLIERS AND LICENSORS, BE LIABLE FOR ANY LOST REVENUE, LOST PROFIT, OR LOST OR DAMAGED DATA, BUSINESS INTERRUPTION, LOSS OF CAPITAL, OR FOR SPECIAL, INDIRECT, CONSEQUENTIAL, INCIDENTAL, OR PUNITIVE DAMAGES, HOWSOEVER ARISING, INCLUDING, WITHOUT LIMITATION, IN CONTRACT, TORT (INCLUDING NEGLIGENCE) OR WHETHER ARISING OUT OF THE USE OF OR INABILITY TO USE THE SOFTWARE, EVEN IF, IN EACH CASE, CISCO, ITS AFFILIATES, OFFICERS, DIRECTORS, EMPLOYEES, AGENTS, SUPPLIERS AND LICENSORS, HAVE BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. BECAUSE SOME STATES OR JURISDICTIONS DO NOT ALLOW LIMITATION OR EXCLUSION OF CONSEQUENTIAL OR INCIDENTAL DAMAGES, THE ABOVE LIMITATION MAY NOT FULLY APPLY TO YOU. THE FOREGOING EXCLUSION SHALL NOT APPLY TO ANY LIABILITY ARISING OUT OF OR IN " + + "CONNECTION WITH: (I) DEATH OR PERSONAL INJURY, (II) FRAUDULENT MISREPRESENTATION, OR (III) CISCO'S LIABILITY IN CONNECTION WITH ANY TERMS THAT CANNOT BE EXCLUDED UNDER APPLICABLE LAW.\n" + + "\n" + + "Customer acknowledges and agrees that Cisco has set its prices and entered into the Agreement in reliance upon the disclaimers of warranty and the limitations of liability set forth herein, that the same reflect an allocation of risk between the parties (including the risk that a contract remedy may fail of its essential purpose and cause consequential loss), and that the same form an essential basis of the bargain between the parties.\n" + + "\n" + + "Controlling Law, Jurisdiction. If you acquired, by reference to the address on the purchase order accepted by the Approved Source, the Software in the United States, Latin America, or the Caribbean, the Agreement and warranties (\"Warranties\") are controlled by and construed under the laws of the State of California, United States of America, notwithstanding any conflicts of law provisions; and the state and federal courts of California shall have exclusive jurisdiction over any claim arising under the Agreement or Warranties. If you acquired the Software in Canada, unless expressly prohibited by local law, the Agreement and Warranties are controlled by and construed under the laws of the Province of Ontario, Canada, notwithstanding any conflicts of law provisions; and the courts of the Province of Ontario shall have exclusive jurisdiction over any claim arising under the Agreement or Warranties. If you acquired the Software in " + + "Europe, the Middle East, Africa, Asia or Oceania (excluding Australia), unless expressly prohibited by local law, the Agreement and Warranties are controlled by and construed under the laws of England, notwithstanding any conflicts of law provisions; and the English courts shall have exclusive jurisdiction over any claim arising under the Agreement or Warranties. In addition, if the Agreement is controlled by the laws of England, no person who is not a party to the Agreement shall be entitled to enforce or take the benefit of any of its terms under the Contracts (Rights of Third Parties) Act 1999. If you acquired the Software in Japan, unless expressly prohibited by local law, the Agreement and Warranties are controlled by and construed under the laws of Japan, notwithstanding any conflicts of law provisions; and the Tokyo District Court of Japan shall have exclusive jurisdiction over any claim arising under the Agreement or Warranties. " + + "If you acquired the Software in Australia, unless expressly prohibited by local law, the Agreement and Warranties are controlled by and construed under the laws of the State of New South Wales, Australia, notwithstanding any conflicts of law provisions; and the State and federal courts of New South Wales shall have exclusive jurisdiction over any claim arising under the Agreement or Warranties. If you acquired the Software in any other country, unless expressly prohibited by local law, the Agreement and Warranties are controlled by and construed under the laws of the State of California, United States of America, notwithstanding any conflicts of law provisions; and the state and federal courts of California shall have exclusive jurisdiction over any claim arising under the Agreement or Warranties.\n" + + "\n" + + "For all countries referred to above, the parties specifically disclaim the application of the UN Convention on Contracts for the International Sale of Goods. Notwithstanding the foregoing, either party may seek interim injunctive relief in any court of appropriate jurisdiction with respect to any alleged breach of such party's intellectual property or proprietary rights. If any portion hereof is found to be void or unenforceable, the remaining provisions of the Agreement and Warranties shall remain in full force and effect. Except as expressly provided herein, the Agreement constitutes the entire agreement between the parties with respect to the license of the Software and Documentation and supersedes any conflicting or additional terms contained in any Purchase Order or elsewhere, all of which terms are excluded. The Agreement has been written in the English language, and the parties agree that the English version will govern.\n" + + "\n" + + "Product warranty terms and other information applicable to Cisco products are available at the following URL: www.cisco.com/go/warranty\n" + + "\n" + + "Cisco and the Cisco logo are trademarks or registered trademarks of Cisco and/or its affiliates in the U.S. and other countries. To view a list of Cisco trademarks, go to this URL: www.cisco.com/go/trademarks. Third-party trademarks mentioned are the property of their respective owners. The use of the word partner does not imply a partnership relationship between Cisco and any other company. (1110R)\n" + + "\n" + + "© 1998, 2001, 2003, 2008-2014 Cisco Systems, Inc. All rights reserved.\n" + + "\n" + + "\n" + + " supplemental end-user license agreement\n" + + " SUPPLEMENTAL END USER LICENSE AGREEMENT FOR VIRTUAL SOFTWARE PRODUCTS\n" + + "\n" + + "IMPORTANT: READ CAREFULLY\n" + + "\n" + + "This Supplemental End User License Agreement (\"SEULA\") contains additional terms and conditions for the Software licensed under the End User License Agreement (\"EULA\") between you and Cisco (collectively, the \"Agreement\"). Capitalized terms used in this SEULA but not defined will have the meanings assigned to them in the EULA. To the extent that there is a conflict between the terms and conditions of the EULA and this SEULA, the terms and conditions of this SEULA will take precedence. In addition to the limitations set forth in the EULA on your access and use of the Software, you agree to comply at all times with the terms and conditions provided in this SEULA.\n" + + "\n" + + "DOWNLOADING, INSTALLING, OR USING THE SOFTWARE CONSTITUTES ACCEPTANCE OF THE AGREEMENT, AND YOU ARE BINDING YOURSELF AND THE BUSINESS ENTITY THAT YOU REPRESENT (COLLECTIVELY, \"CUSTOMER\") TO THE AGREEMENT. IF YOU DO NOT AGREE TO ALL OF THE TERMS OF THE AGREEMENT, THEN CISCO IS UNWILLING TO LICENSE THE SOFTWARE TO YOU AND (A) YOU MAY NOT DOWNLOAD, INSTALL OR USE THE SOFTWARE, AND (B) YOU MAY RETURN THE SOFTWARE (INCLUDING ANY UNOPENED CD PACKAGE AND ANY WRITTEN MATERIALS) FOR A FULL REFUND, OR, IF THE SOFTWARE AND WRITTEN MATERIALS ARE SUPPLIED AS PART OF ANOTHER PRODUCT, YOU MAY RETURN THE ENTIRE PRODUCT FOR A FULL REFUND. YOUR RIGHT TO RETURN AND REFUND EXPIRES 30 DAYS AFTER PURCHASE FROM CISCO OR AN AUTHORIZED CISCO RESELLER, AND APPLIES ONLY IF YOU ARE THE ORIGINAL END USER PURCHASER.\n" + + "\n" + + "Definitions\n" + + "\"CPU\" means a central processing unit that encompasses part of a Server.\n" + + "\"Failover Pair\" means a primary Instance and a standby Instance with the same Software configuration where the standby Instance can take over in case of failure of the primary Instance.\n" + + "\"Instance\" means a single copy of the Software. Each copy of the Software loaded into memory is an Instance.\n" + + "\"Server\" means a single physical computer or device on a network that manages or provides network resources for multiple users.\n" + + "\"Service Provider\" means a company that provides information technology services to external end user customers.\n" + + "\"Software\" means Cisco's Adaptive Security Virtual Appliance (\"ASAv\"), Adaptive Security Appliance 1000V Cloud Firewall Software (\"ASA 1000V\"), Nexus 1000V series switch products, Virtual Security Gateway products, or other Cisco virtual software products that Cisco includes under this SEULA.\n" + + "\"vCPU\" means a virtual central processing resource assigned to the VM by the underlying virtualization technology.\n" + + "\"Virtual Machine\" or \"VM\" means a software container that can run its own operating system and execute applications like a Server.\n" + + "\n" + + "Additional License Terms and Conditions\n" + + "1. Cisco hereby grants Customer the right to install and use the Software on single or multiple Cisco or non-Cisco Servers or on Virtual Machines. In order to use the Software Customer may be required to input a registration number or product activation key and register each Instance online at Cisco's website in order to obtain the necessary entitlements.\n" + + "2. Customer shall pay a unit license fee to Cisco or an authorized Cisco reseller, as applicable, for each Instance installed on a Cisco or non-Cisco Server CPU, vCPU or Virtual Machine, as determined by Cisco.\n" + + "3. For the ASA 1000V, Customer is licensed the number of Instances equal to the number of CPUs covered by the unit license fee. If Customer deploys a Failover Pair, then the fee for the additional standby Instance is included in the fee for each primary Instance.\n" + + "4. If Customer is a Service Provider, Customer may use the Software under the terms of this Agreement for the purpose of delivering hosted information technology services to Customer's end user customers, subject to payment of the required license fee(s).\n" + + "5. Customer may also use the Software under the terms of this Agreement to deliver hosted information technology services to Customer affiliates, subject to payment of the required license fee(s).\n" + + "6. If the Software is subject to Cisco's Smart Licensing program, Cisco will be able to assess if Customer is using the Software within the limits and entitlements paid for by Customer. If the Smart Licensing program is applicable, Customer will be required to enter into a separate terms of service agreement relating to Smart Licensing.\n" + + "\n" + + ""; + + private String productSectionWithCategories = + "\n" + + "\n" + + " Appliance ISV branding information\n" + + " VMware vCenter Server Appliance\n" + + " VMware Inc.\n" + + " \n" + + "\n" + + "\n" + + " 6.7.0.44000\n" + + " 6.7.0.44000 build 16046470\n" + + " \n" + + " http://www.vmware.com\n" + + " https://${vami.ip0.VMware_vCenter_Server_Appliance}:5480/\n" + + " Application\n" + + " Networking Configuration\n" + + " \n" + + " \n" + + " Network IP address family (i.e., 'ipv4' or 'ipv6').\n" + + " \n" + + " \n" + + " \n" + + " Network mode (i.e., 'static', 'dhcp', or 'autoconf' (IPv6 only).\n" + + " \n" + + " \n" + + " \n" + + " Network IP address. Only provide this when mode is 'static'. Can be IPv4 or IPv6 based on specified address family.\n" + + " \n" + + " \n" + + " \n" + + " Network prefix length. Only provide this when mode is 'static'. 0-32 for IPv4. 0-128 for IPv6.\n" + + " \n" + + " \n" + + " \n" + + " IP address of default gateway. Can be 'default' when using IPv6.\n" + + " \n" + + " \n" + + " \n" + + " Comma separated list of IP addresses of DNS servers.\n" + + " \n" + + " \n" + + " \n" + + " Network identity (IP address or fully-qualified domain name) services should use when advertising themselves.\n" + + " \n" + + " \n" + + " \n" + + " A string encoding a JSON object mapping port names to port numbers.\n" + + " \n" + + " SSO Configuration\n" + + " \n" + + " \n" + + " For the first instance of the identity domain, this is the username with Administrator privileges. Otherwise, this is the username of the replication partner.\n" + + " \n" + + " \n" + + " \n" + + " For the first instance of the identity domain, this is the password given to the Administrator account. Otherwise, this is the password of the Administrator account of the replication partner.\n" + + " \n" + + " \n" + + " \n" + + " For the first instance of the identity domain, this is the name of the newly created domain.\n" + + " \n" + + " \n" + + " \n" + + " Name of site. Use 'Default-First-Site' to define a new site.\n" + + " \n" + + " \n" + + " \n" + + " If this parameter is set to True, the VMware directory instance is setup as the first instance of a new identity domain. Otherwise, the instance is setup as a replication partner.\n" + + " \n" + + " \n" + + " \n" + + " The hostname of the VMware directory replication partner. This value is ignored for the first instance of the identity domain.\n" + + " \n" + + " Database Configuration\n" + + " \n" + + " \n" + + " String indicating whether the database is 'embedded' or 'external'.\n" + + " \n" + + " \n" + + " \n" + + " String naming the account to use when connecting to external database (ignored when db.type is 'embedded').\n" + + " \n" + + " \n" + + " \n" + + " String providing the password to use when connecting to external database (ignored when db.type is 'embedded').\n" + + " \n" + + " \n" + + " \n" + + " String naming the the hostname of the server on which the external database is running (ignored when db.type is 'embedded').\n" + + " \n" + + " \n" + + " \n" + + " String describing the port on the host on which the external database is running (ignored when db.type is 'embedded').\n" + + " \n" + + " \n" + + " \n" + + " String describing the external database provider. The only supported value is 'oracle' (ignored when the db.type is 'embedded').\n" + + " \n" + + " \n" + + " \n" + + " String describing the external database instance. Values could be anything depending on what the database instance name the DBA creates in the external db. (ignored when the db.type is 'embedded').\n" + + " \n" + + " System Configuration\n" + + " \n" + + " \n" + + " Password to assign to root account. If blank, password can be set on the console.\n" + + " \n" + + " \n" + + " \n" + + " This property is not changeable.\n" + + " \n" + + " \n" + + " \n" + + " Set whether SSH-based remote login is enabled. This configuration can be changed after deployment.\n" + + " \n" + + " \n" + + " \n" + + " Set whether VMware tools based time synchronization should be used. This parameter is ignored if appliance.ntp.servers is not empty.\n" + + " \n" + + " \n" + + " \n" + + " A comma-seperated list of hostnames or IP addresses of NTP Servers\n" + + " \n" + + " \n" + + " \n" + + " Type of appliance to deploy (i.e. 'embedded', 'infrastructure' or 'management').\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " When deploying a vCenter Server Node, please provide the FQDN or IP address of a Platform Services Controller (leave blank otherwise). The choice of FQDN versus IP address is decided based on the Platform Services Controller's own notion of its network identity.\n" + + " \n" + + " \n" + + " \n" + + " When deploying a vCenter Server pointing to an external platform services controller, please provide the HTTPS port of the external platform services controller if a custom port number is being used. The default HTTPS port number is 443.\n" + + " \n" + + " Upgrade Configuration\n" + + " \n" + + " \n" + + " IP/hostname of the appliance to upgrade. Set only for upgrade.\n" + + " \n" + + " \n" + + " \n" + + " Port used by Migration Assistant on source vCenter Server.\n" + + " \n" + + " \n" + + " \n" + + " vCenter username for the appliance to upgrade. Set only for upgrade.\n" + + " \n" + + " \n" + + " \n" + + " vCenter password for the appliance to upgrade. Set only for upgrade.\n" + + " \n" + + " \n" + + " \n" + + " Username for the appliance operating system to upgrade. Usually root. Set only for upgrade.\n" + + " \n" + + " \n" + + " \n" + + " Password for the appliance operating system to upgrade. Set only for upgrade.\n" + + " \n" + + " \n" + + " \n" + + " URL that consists of the IP address or FQDN and https port of the vCenter Server instance or ESXi host that manages the appliance to upgrade. Https port is an optional parameter which by default is 443. Example: 10.10.10.10, //10.10.10.10:444, //[2001:db8:a0b:12f0::1]:444. Set only for upgrade.\n" + + " \n" + + " \n" + + " \n" + + " Username for the host that manages appliance to upgrade. Can be either vCenter or ESX host. Set only for upgrade.\n" + + " \n" + + " \n" + + " \n" + + " Password for the host that manages appliance to upgrade. Can be either vCenter or ESX host. Set only for upgrade.\n" + + " \n" + + " \n" + + " \n" + + " Thumbprint for the SSL certificate of the host that manages the appliance to upgrade. Set only for upgrade.\n" + + " \n" + + " \n" + + " \n" + + " Source host platform. Optional. Set only for upgrade\n" + + " \n" + + " \n" + + " \n" + + " Folder on the source appliance, where to store migrate data. Optional. Set only for upgrade\n" + + " \n" + + " \n" + + " \n" + + " Folder where exported source data will be stored in the appliance. Optional. Set only for upgrade\n" + + " \n" + + " \n" + + " \n" + + " Advanced upgrade settings specified in json format. Optional. Set only for upgrade\n" + + " \n" + + " \n" + + " \n" + + " Active Directory domain to join.\n" + + " \n" + + " \n" + + " \n" + + " Active Directory domain admin user. This username will be used to join the machine to the domain.\n" + + " \n" + + " \n" + + " \n" + + " Active Directory domain admin user password. This password will be used to join the machine to the domain.\n" + + " \n" + + " \n" + + " \n" + + " FQDN or IP address of the vCenter Server managing that target appliance. Used when upgrading a source appliance in VCHA cluster.\n" + + " \n" + + " \n" + + " \n" + + " Https port of the vCenter Server managing that target appliance. Used when upgrading a source appliance in VCHA cluster. If not specified, port 443 will be used by default.\n" + + " \n" + + " \n" + + " \n" + + " User able to authenticate in vCenter Server managing that target appliance. The user must have the privilege Global.VCServer. Used when upgrading a source appliance in VCHA cluster.\n" + + " \n" + + " \n" + + " \n" + + " Password for administrator user authenticating to the vCenter Server managing target appliance. Used when upgrading a source appliance in VCHA cluster.\n" + + " \n" + + " \n" + + " \n" + + " Thumbprint for the SSL certificate of the host that manages the appliance to upgrade. Used when upgrading a source appliance in VCHA cluster.\n" + + " \n" + + " \n" + + " \n" + + " Path to host/cluster/resource pool where target appliance will be deployed on management vCenter Server. Used when upgrading a source appliance in VCHA cluster. Example: /my_datacenter/my_folder/my_host_or_cluster/my_resource_pool\n" + + " \n" + + " Miscellaneous\n" + + " \n" + + " \n" + + " Set whether ESXi Dump Collector service is enabled. This configuration can be changed after deployment.\n" + + " \n" + + " \n" + + " \n" + + " If this parameter is set to True, no questions will be posted during install or upgrade. Otherwise, the install process will wait for a reply if there is a pending question.\n" + + " \n" + + " \n" + + " \n" + + " This parameter specifies the client locale. Supported locales are en, fr, ja, ko, zh_CN and zh_TW. English is assumed if locale is unknown.\n" + + " \n" + + " \n" + + " \n" + + " Specify feature switch states which need to be added or modified in feature switch state config file. Format: key1=value1, key2=value2\n" + + " \n" + + " \n" + + " \n" + + " VMware’s Customer Experience Improvement Program ("CEIP") provides VMware with information that enables VMware to improve its products and services, to fix problems, and to advise you on how best to deploy and use our products. As part of the CEIP, VMware collects technical information about your organization’s use of VMware products and services on a regular basis in association with your organization’s VMware license key(s). This information does not personally identify any individual. For more details about the Program and how VMware uses the information it collects through CEIP, please see the product documentation at http://www.vmware.com/info?id=1399. If you want to participate in VMware’s CEIP for this product, set this property to True. You may join or leave VMware’s CEIP for this product at any time.\n" + + " \n" + + " \n" + + " \n" + + " If this parameter is set to True, the appliance will be configured after deployment using the specified OVF configuration parameters. If set to False, the appliance should be configured post-deployment using the VMware Appliance Management Interface.\n" + + " \n" + + " \n" + + " \n" + + " If a valid MAC address prefix is provided, then all MAC addresses assigned by vCenter Server will begin with this prefix instead of the VMware OUI. This property cannot co-exist with mac-allocation-scheme.ranges\n" + + " \n" + + " \n" + + " \n" + + " This property is mandatory whenever a custom MAC prefix is provided.\n" + + " \n" + + " \n" + + " \n" + + " If valid MAC address range is provided, then vCenter Server will assign MAC addresses from this range instead of allocating VMware OUI based MAC address. The address range must be provided in the format "BeginAddress1-EndAddress1,...,BeginAddressN-EndAddressN". This property cannot co-exist with mac-allocation-scheme.prefix.\n" + + " \n" + + "\n" + + "\n" + + " VAMI Properties\n" + + " Networking Properties\n" + + " \n" + + " \n" + + " The domain name of this VM. Leave blank if DHCP is desired.\n" + + " \n" + + " \n" + + " \n" + + " The domain search path (comma or space separated domain names) for this VM. Leave blank if DHCP is desired.\n" + + " \n" + + "\n" + + "\n" + + " VM specific properties\n" + + " \n" + + "\n" + + ""; + private OVFHelper ovfHelper = new OVFHelper(); @Test public void testGetOVFPropertiesValidOVF() throws IOException, SAXException, ParserConfigurationException { - List props = ovfHelper.getOVFPropertiesXmlString(ovfFileProductSection); + List props = ovfHelper.getOVFPropertiesFromXmlString(ovfFileProductSection); Assert.assertEquals(2, props.size()); } @Test(expected = SAXParseException.class) public void testGetOVFPropertiesInvalidOVF() throws IOException, SAXException, ParserConfigurationException { - ovfHelper.getOVFPropertiesXmlString(ovfFileProductSection + "xxxxxxxxxxxxxxxxx"); + ovfHelper.getOVFPropertiesFromXmlString(ovfFileProductSection + "xxxxxxxxxxxxxxxxx"); + } + + @Test + public void testGetOVFDeploymentOptionsValidOVF() throws IOException, SAXException, ParserConfigurationException { + List options = ovfHelper.getOVFDeploymentOptionsFromXmlString(ovfFileDeploymentOptionsSection); + Assert.assertEquals(3, options.size()); + } + + @Test + public void testGetOVFVirtualHardwareSectionValidOVF() throws IOException, SAXException, ParserConfigurationException { + List items = ovfHelper.getOVFVirtualHardwareSectionFromXmlString(ovfFileVirtualHardwareSection); + Assert.assertEquals(20, items.size()); + } + + @Test + public void testGetOVFEulaSectionValidOVF() throws IOException, SAXException, ParserConfigurationException { + List eulas = ovfHelper.getOVFEulaSectionFromXmlString(eulaSections); + Assert.assertEquals(2, eulas.size()); + } + + @Test + public void testGetOVFPropertiesWithCategories() throws IOException, SAXException, ParserConfigurationException { + List props = ovfHelper.getOVFPropertiesFromXmlString(productSectionWithCategories); + Assert.assertEquals(18, props.size()); } } diff --git a/core/pom.xml b/core/pom.xml index ff63e50c4a0..d33d6866f7f 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -51,9 +51,5 @@ commons-codec commons-codec - - org.apache.commons - commons-compress - diff --git a/core/src/main/java/com/cloud/agent/api/storage/DownloadAnswer.java b/core/src/main/java/com/cloud/agent/api/storage/DownloadAnswer.java index 9859c3f83d0..a7b9179e200 100644 --- a/core/src/main/java/com/cloud/agent/api/storage/DownloadAnswer.java +++ b/core/src/main/java/com/cloud/agent/api/storage/DownloadAnswer.java @@ -25,8 +25,10 @@ import java.util.List; import com.cloud.agent.api.Answer; import com.cloud.agent.api.Command; import com.cloud.agent.api.LogLevel; +import com.cloud.agent.api.to.DatadiskTO; import com.cloud.storage.VMTemplateStorageResourceAssoc; import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; +import org.apache.cloudstack.api.net.NetworkPrerequisiteTO; public class DownloadAnswer extends Answer { private String jobId; @@ -38,8 +40,17 @@ public class DownloadAnswer extends Answer { private long templateSize = 0L; private long templatePhySicalSize = 0L; private String checkSum; + @LogLevel(LogLevel.Log4jLevel.Off) private List ovfProperties; + @LogLevel(LogLevel.Log4jLevel.Off) + private List networkRequirements; + @LogLevel(LogLevel.Log4jLevel.Off) + private List disks; + @LogLevel(LogLevel.Log4jLevel.Off) + private OVFVirtualHardwareSectionTO ovfHardwareSection; + @LogLevel(LogLevel.Log4jLevel.Off) + private List eulaSections; public String getCheckSum() { return checkSum; @@ -157,4 +168,36 @@ public class DownloadAnswer extends Answer { public void setOvfProperties(List ovfProperties) { this.ovfProperties = ovfProperties; } + + public List getNetworkRequirements() { + return networkRequirements; + } + + public void setNetworkRequirements(List networkRequirements) { + this.networkRequirements = networkRequirements; + } + + public List getDisks() { + return disks; + } + + public void setDisks(List disks) { + this.disks = disks; + } + + public OVFVirtualHardwareSectionTO getOvfHardwareSection() { + return ovfHardwareSection; + } + + public void setOvfHardwareSection(OVFVirtualHardwareSectionTO ovfHardwareSection) { + this.ovfHardwareSection = ovfHardwareSection; + } + + public List getEulaSections() { + return eulaSections; + } + + public void setEulaSections(List eulaSections) { + this.eulaSections = eulaSections; + } } diff --git a/core/src/main/java/com/cloud/agent/api/storage/GetDatadisksCommand.java b/core/src/main/java/com/cloud/agent/api/storage/GetDatadisksCommand.java index 0e22ea25e78..a6dfbfef99a 100644 --- a/core/src/main/java/com/cloud/agent/api/storage/GetDatadisksCommand.java +++ b/core/src/main/java/com/cloud/agent/api/storage/GetDatadisksCommand.java @@ -21,10 +21,12 @@ import com.cloud.agent.api.to.DataTO; public final class GetDatadisksCommand extends Command { private DataTO data; + private String configurationId; - public GetDatadisksCommand(DataTO data) { + public GetDatadisksCommand(DataTO data, String configurationId) { super(); this.data = data; + this.configurationId = configurationId; } protected GetDatadisksCommand() { @@ -40,4 +42,7 @@ public final class GetDatadisksCommand extends Command { return data; } + public String getConfigurationId() { + return configurationId; + } } \ No newline at end of file diff --git a/core/src/main/java/com/cloud/resource/ServerResource.java b/core/src/main/java/com/cloud/resource/ServerResource.java index 9030db72f00..16ac00ed176 100644 --- a/core/src/main/java/com/cloud/resource/ServerResource.java +++ b/core/src/main/java/com/cloud/resource/ServerResource.java @@ -31,6 +31,7 @@ import com.cloud.utils.component.Manager; * ServerResource is a generic container to execute commands sent */ public interface ServerResource extends Manager { + /** * @return Host.Type type of the computing server we have. */ diff --git a/core/src/main/java/com/cloud/storage/resource/StorageProcessor.java b/core/src/main/java/com/cloud/storage/resource/StorageProcessor.java index 4b2438ea102..d86a1a619a9 100644 --- a/core/src/main/java/com/cloud/storage/resource/StorageProcessor.java +++ b/core/src/main/java/com/cloud/storage/resource/StorageProcessor.java @@ -34,6 +34,9 @@ import org.apache.cloudstack.storage.command.SnapshotAndCopyCommand; import com.cloud.agent.api.Answer; public interface StorageProcessor { + + String REQUEST_TEMPLATE_RELOAD = "request template reload"; + public Answer copyTemplateToPrimaryStorage(CopyCommand cmd); public Answer cloneVolumeFromBaseTemplate(CopyCommand cmd); diff --git a/core/src/main/java/com/cloud/storage/template/OVAProcessor.java b/core/src/main/java/com/cloud/storage/template/OVAProcessor.java index d771c67acec..794ac48c904 100644 --- a/core/src/main/java/com/cloud/storage/template/OVAProcessor.java +++ b/core/src/main/java/com/cloud/storage/template/OVAProcessor.java @@ -20,13 +20,20 @@ package com.cloud.storage.template; import java.io.File; +import java.io.IOException; import java.util.List; import java.util.Map; import javax.naming.ConfigurationException; import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import com.cloud.agent.api.storage.OVFConfigurationTO; +import com.cloud.agent.api.storage.OVFEulaSectionTO; import com.cloud.agent.api.storage.OVFPropertyTO; +import com.cloud.agent.api.storage.OVFVirtualHardwareItemTO; +import com.cloud.agent.api.storage.OVFVirtualHardwareSectionTO; +import org.apache.cloudstack.api.net.NetworkPrerequisiteTO; import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; import org.w3c.dom.Document; @@ -41,9 +48,13 @@ import com.cloud.storage.StorageLayer; import com.cloud.utils.Pair; import com.cloud.utils.component.AdapterBase; import com.cloud.utils.script.Script; +import org.xml.sax.SAXException; +/** + * processes the content of an OVA for registration of a template + */ public class OVAProcessor extends AdapterBase implements Processor { - private static final Logger s_logger = Logger.getLogger(OVAProcessor.class); + private static final Logger LOGGER = Logger.getLogger(OVAProcessor.class); StorageLayer _storage; @Override @@ -53,73 +64,137 @@ public class OVAProcessor extends AdapterBase implements Processor { @Override public FormatInfo process(String templatePath, ImageFormat format, String templateName, long processTimeout) throws InternalErrorException { - if (format != null) { - if (s_logger.isInfoEnabled()) { - s_logger.info("We currently don't handle conversion from " + format + " to OVA."); - } + if (! conversionChecks(format)){ return null; } - s_logger.info("Template processing. templatePath: " + templatePath + ", templateName: " + templateName); + LOGGER.info("Template processing. templatePath: " + templatePath + ", templateName: " + templateName); String templateFilePath = templatePath + File.separator + templateName + "." + ImageFormat.OVA.getFileExtension(); if (!_storage.exists(templateFilePath)) { - if (s_logger.isInfoEnabled()) { - s_logger.info("Unable to find the vmware template file: " + templateFilePath); + if (LOGGER.isInfoEnabled()) { + LOGGER.info("Unable to find the vmware template file: " + templateFilePath); } return null; } - s_logger.info("Template processing - untar OVA package. templatePath: " + templatePath + ", templateName: " + templateName); + String templateFileFullPath = unpackOva(templatePath, templateName, processTimeout); + + setFileSystemAccessRights(templatePath); + + FormatInfo info = createFormatInfo(templatePath, templateName, templateFilePath, templateFileFullPath); + + return info; + } + + private FormatInfo createFormatInfo(String templatePath, String templateName, String templateFilePath, String templateFileFullPath) throws InternalErrorException { + FormatInfo info = new FormatInfo(); + info.format = ImageFormat.OVA; + info.filename = templateName + "." + ImageFormat.OVA.getFileExtension(); + info.size = _storage.getSize(templateFilePath); + info.virtualSize = getTemplateVirtualSize(templatePath, info.filename); + validateOva(templateFileFullPath, info); + + return info; + } + + /** + * side effect; properties are added to the info + * + * @throws InternalErrorException on an invalid ova contents + */ + private void validateOva(String templateFileFullPath, FormatInfo info) throws InternalErrorException { + String ovfFilePath = getOVFFilePath(templateFileFullPath); + OVFHelper ovfHelper = new OVFHelper(); + Document doc = ovfHelper.getDocumentFromFile(ovfFilePath); + + List disks = ovfHelper.getOVFVolumeInfoFromFile(ovfFilePath, doc, null); + if (LOGGER.isTraceEnabled()) { + LOGGER.trace(String.format("Found %d disks in template %s", CollectionUtils.isNotEmpty(disks) ? disks.size() : 0, ovfFilePath)); + } + if (CollectionUtils.isNotEmpty(disks)) { + info.disks = disks; + } + + List nets = ovfHelper.getNetPrerequisitesFromDocument(doc); + if (CollectionUtils.isNotEmpty(nets)) { + LOGGER.info("Found " + nets.size() + " prerequisite networks"); + info.networks = nets; + } else if (LOGGER.isTraceEnabled()) { + LOGGER.trace(String.format("no net prerequisites found in template %s", ovfFilePath)); + } + + List ovfProperties = ovfHelper.getConfigurableOVFPropertiesFromDocument(doc); + if (CollectionUtils.isNotEmpty(ovfProperties)) { + LOGGER.info("Found " + ovfProperties.size() + " configurable OVF properties"); + info.ovfProperties = ovfProperties; + } else if (LOGGER.isTraceEnabled()) { + LOGGER.trace(String.format("no ovf properties found in template %s", ovfFilePath)); + } + + OVFVirtualHardwareSectionTO hardwareSection = ovfHelper.getVirtualHardwareSectionFromDocument(doc); + List configurations = hardwareSection.getConfigurations(); + if (CollectionUtils.isNotEmpty(configurations)) { + LOGGER.info("Found " + configurations.size() + " deployment option configurations"); + } + List hardwareItems = hardwareSection.getCommonHardwareItems(); + if (CollectionUtils.isNotEmpty(hardwareItems)) { + LOGGER.info("Found " + hardwareItems.size() + " virtual hardware items"); + } + info.hardwareSection = hardwareSection; + List eulaSections = ovfHelper.getEulaSectionsFromDocument(doc); + if (CollectionUtils.isNotEmpty(eulaSections)) { + LOGGER.info("Found " + eulaSections.size() + " license agreements"); + info.eulaSections = eulaSections; + } + } + + private void setFileSystemAccessRights(String templatePath) { + Script command; + String result; + + command = new Script("chmod", 0, LOGGER); + command.add("-R"); + command.add("666", templatePath); + result = command.execute(); + if (result != null) { + LOGGER.warn("Unable to set permissions for files in " + templatePath + " due to " + result); + } + command = new Script("chmod", 0, LOGGER); + command.add("777", templatePath); + result = command.execute(); + if (result != null) { + LOGGER.warn("Unable to set permissions for " + templatePath + " due to " + result); + } + } + + private String unpackOva(String templatePath, String templateName, long processTimeout) throws InternalErrorException { + LOGGER.info("Template processing - untar OVA package. templatePath: " + templatePath + ", templateName: " + templateName); String templateFileFullPath = templatePath + File.separator + templateName + "." + ImageFormat.OVA.getFileExtension(); File templateFile = new File(templateFileFullPath); - Script command = new Script("tar", processTimeout, s_logger); + Script command = new Script("tar", processTimeout, LOGGER); command.add("--no-same-owner"); command.add("--no-same-permissions"); command.add("-xf", templateFileFullPath); command.setWorkDir(templateFile.getParent()); String result = command.execute(); if (result != null) { - s_logger.info("failed to untar OVA package due to " + result + ". templatePath: " + templatePath + ", templateName: " + templateName); + LOGGER.info("failed to untar OVA package due to " + result + ". templatePath: " + templatePath + ", templateName: " + templateName); throw new InternalErrorException("failed to untar OVA package"); } + return templateFileFullPath; + } - command = new Script("chmod", 0, s_logger); - command.add("-R"); - command.add("666", templatePath); - result = command.execute(); - if (result != null) { - s_logger.warn("Unable to set permissions for files in " + templatePath + " due to " + result); - } - command = new Script("chmod", 0, s_logger); - command.add("777", templatePath); - result = command.execute(); - if (result != null) { - s_logger.warn("Unable to set permissions for " + templatePath + " due to " + result); - } - - FormatInfo info = new FormatInfo(); - info.format = ImageFormat.OVA; - info.filename = templateName + "." + ImageFormat.OVA.getFileExtension(); - info.size = _storage.getSize(templateFilePath); - info.virtualSize = getTemplateVirtualSize(templatePath, info.filename); - - //vaidate ova - String ovfFile = getOVFFilePath(templateFileFullPath); - try { - OVFHelper ovfHelper = new OVFHelper(); - List disks = ovfHelper.getOVFVolumeInfo(ovfFile); - List ovfProperties = ovfHelper.getOVFPropertiesFromFile(ovfFile); - if (CollectionUtils.isNotEmpty(ovfProperties)) { - s_logger.info("Found " + ovfProperties.size() + " configurable OVF properties"); - info.ovfProperties = ovfProperties; + private boolean conversionChecks(ImageFormat format) { + if (format != null) { + if (LOGGER.isInfoEnabled()) { + LOGGER.info("We currently don't handle conversion from " + format + " to OVA."); } - } catch (Exception e) { - s_logger.info("The ovf file " + ovfFile + " is invalid ", e); - throw new InternalErrorException("OVA package has bad ovf file " + e.getMessage(), e); + return false; } - // delete original OVA file - // templateFile.delete(); - return info; + if (LOGGER.isTraceEnabled()) { + LOGGER.trace("We are handling format " + format + "."); + } + return true; } @Override @@ -128,34 +203,43 @@ public class OVAProcessor extends AdapterBase implements Processor { long size = getTemplateVirtualSize(file.getParent(), file.getName()); return size; } catch (Exception e) { - s_logger.info("[ignored]" + LOGGER.info("[ignored]" + "failed to get virtual template size for ova: " + e.getLocalizedMessage()); } return file.length(); } + /** + * gets the virtual size from the OVF file meta data. + * + * @return the accumulative virtual size of the disk definitions in the OVF + * @throws InternalErrorException + */ public long getTemplateVirtualSize(String templatePath, String templateName) throws InternalErrorException { - // get the virtual size from the OVF file meta data long virtualSize = 0; String templateFileFullPath = templatePath.endsWith(File.separator) ? templatePath : templatePath + File.separator; templateFileFullPath += templateName.endsWith(ImageFormat.OVA.getFileExtension()) ? templateName : templateName + "." + ImageFormat.OVA.getFileExtension(); String ovfFileName = getOVFFilePath(templateFileFullPath); if (ovfFileName == null) { String msg = "Unable to locate OVF file in template package directory: " + templatePath; - s_logger.error(msg); + LOGGER.error(msg); throw new InternalErrorException(msg); } try { Document ovfDoc = null; ovfDoc = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(new File(ovfFileName)); - Element disk = (Element)ovfDoc.getElementsByTagName("Disk").item(0); - virtualSize = Long.parseLong(disk.getAttribute("ovf:capacity")); - String allocationUnits = disk.getAttribute("ovf:capacityAllocationUnits"); - virtualSize = OVFHelper.getDiskVirtualSize(virtualSize, allocationUnits, ovfFileName); + NodeList diskElements = ovfDoc.getElementsByTagName("Disk"); + for (int i = 0; i < diskElements.getLength(); i++) { + Element disk = (Element)diskElements.item(i); + long diskSize = Long.parseLong(disk.getAttribute("ovf:capacity")); + String allocationUnits = disk.getAttribute("ovf:capacityAllocationUnits"); + diskSize = OVFHelper.getDiskVirtualSize(diskSize, allocationUnits, ovfFileName); + virtualSize += diskSize; + } return virtualSize; - } catch (Exception e) { + } catch (InternalErrorException | IOException | NumberFormatException | ParserConfigurationException | SAXException e) { String msg = "getTemplateVirtualSize: Unable to parse OVF XML document " + templatePath + " to get the virtual disk " + templateName + " size due to " + e; - s_logger.error(msg); + LOGGER.error(msg); throw new InternalErrorException(msg); } } @@ -187,9 +271,9 @@ public class OVAProcessor extends AdapterBase implements Processor { } } return new Pair(virtualSize, fileSize); - } catch (Exception e) { + } catch (InternalErrorException | IOException | NumberFormatException | ParserConfigurationException | SAXException e) { String msg = "getDiskDetails: Unable to parse OVF XML document " + ovfFilePath + " to get the virtual disk " + diskName + " size due to " + e; - s_logger.error(msg); + LOGGER.error(msg); throw new InternalErrorException(msg); } } @@ -218,4 +302,4 @@ public class OVAProcessor extends AdapterBase implements Processor { return true; } -} +} \ No newline at end of file diff --git a/core/src/main/java/com/cloud/storage/template/Processor.java b/core/src/main/java/com/cloud/storage/template/Processor.java index 4bb714a7ab9..53fa6b76d04 100644 --- a/core/src/main/java/com/cloud/storage/template/Processor.java +++ b/core/src/main/java/com/cloud/storage/template/Processor.java @@ -23,10 +23,14 @@ import java.io.File; import java.io.IOException; import java.util.List; +import com.cloud.agent.api.storage.OVFEulaSectionTO; import com.cloud.agent.api.storage.OVFPropertyTO; +import com.cloud.agent.api.storage.OVFVirtualHardwareSectionTO; +import com.cloud.agent.api.to.DatadiskTO; import com.cloud.exception.InternalErrorException; import com.cloud.storage.Storage.ImageFormat; import com.cloud.utils.component.Adapter; +import org.apache.cloudstack.api.net.NetworkPrerequisiteTO; /** * Generic interface to process different types of image formats @@ -48,13 +52,17 @@ public interface Processor extends Adapter { FormatInfo process(String templatePath, ImageFormat format, String templateName, long processTimeout) throws InternalErrorException; - public static class FormatInfo { + class FormatInfo { public ImageFormat format; public long size; public long virtualSize; public String filename; public boolean isCorrupted; public List ovfProperties; + public List networks; + public List disks; + public OVFVirtualHardwareSectionTO hardwareSection; + public List eulaSections; } long getVirtualSize(File file) throws IOException; diff --git a/core/src/main/java/org/apache/cloudstack/storage/to/TemplateObjectTO.java b/core/src/main/java/org/apache/cloudstack/storage/to/TemplateObjectTO.java index cc2eaadea07..b184a74312b 100644 --- a/core/src/main/java/org/apache/cloudstack/storage/to/TemplateObjectTO.java +++ b/core/src/main/java/org/apache/cloudstack/storage/to/TemplateObjectTO.java @@ -47,6 +47,8 @@ public class TemplateObjectTO implements DataTO { private boolean bootable; private String uniqueName; private boolean directDownload; + private boolean deployAsIs; + private String deployAsIsConfiguration; public TemplateObjectTO() { @@ -82,6 +84,8 @@ public class TemplateObjectTO implements DataTO { this.imageDataStore = template.getDataStore().getTO(); } this.hypervisorType = template.getHypervisorType(); + this.deployAsIs = template.isDeployAsIs(); + this.deployAsIsConfiguration = template.getDeployAsIsConfiguration(); } @Override @@ -244,6 +248,18 @@ public class TemplateObjectTO implements DataTO { this.directDownload = directDownload; } + public boolean isDeployAsIs() { + return deployAsIs; + } + + public String getDeployAsIsConfiguration() { + return deployAsIsConfiguration; + } + + public void setDeployAsIsConfiguration(String deployAsIsConfiguration) { + this.deployAsIsConfiguration = deployAsIsConfiguration; + } + @Override public String toString() { return new StringBuilder("TemplateTO[id=").append(id).append("|origUrl=").append(origUrl).append("|name").append(name).append("]").toString(); diff --git a/core/src/test/java/com/cloud/agent/api/storage/DownloadAnswerTest.java b/core/src/test/java/com/cloud/agent/api/storage/DownloadAnswerTest.java new file mode 100644 index 00000000000..62bb3d65c83 --- /dev/null +++ b/core/src/test/java/com/cloud/agent/api/storage/DownloadAnswerTest.java @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.api.storage; + +import com.cloud.agent.api.Answer; +import com.cloud.serializer.GsonHelper; +import com.cloud.storage.VMTemplateStorageResourceAssoc; +import com.google.gson.Gson; +import org.apache.cloudstack.api.net.NetworkPrerequisiteTO; +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +public class DownloadAnswerTest { + Gson gson = GsonHelper.getGson(); + + VMTemplateStorageResourceAssoc.Status status = VMTemplateStorageResourceAssoc.Status.DOWNLOADED; + DownloadAnswer answer = new DownloadAnswer("nothin wrong", status); + + @Test + public void redeserialise () + { + String json = gson.toJson(answer); + DownloadAnswer received = gson.fromJson(json, DownloadAnswer.class); + Assert.assertEquals(received,answer); + } + @Test + public void properties () + { + List properties = new ArrayList<>(); + properties.add(new OVFPropertyTO()); + List networks = new ArrayList<>(); + networks.add(new NetworkPrerequisiteTO()); + + answer.setOvfProperties(properties); + answer.setNetworkRequirements(networks); + + String json = gson.toJson(answer); + Answer received = gson.fromJson(json, Answer.class); + Assert.assertEquals(received,answer); + } +} \ No newline at end of file diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java index db13c1f48a8..9baea60f29f 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java @@ -18,6 +18,7 @@ */ package org.apache.cloudstack.engine.orchestration.service; +import java.util.List; import java.util.Map; import java.util.Set; @@ -119,8 +120,11 @@ public interface VolumeOrchestrationService { boolean canVmRestartOnAnotherServer(long vmId); - DiskProfile allocateTemplatedVolume(Type type, String name, DiskOffering offering, Long rootDisksize, Long minIops, Long maxIops, VirtualMachineTemplate template, VirtualMachine vm, - Account owner); + /** + * Allocate a volume or multiple volumes in case of template is registered with the 'deploy-as-is' option, allowing multiple disks + */ + List allocateTemplatedVolumes(Type type, String name, DiskOffering offering, Long rootDisksize, Long minIops, Long maxIops, VirtualMachineTemplate template, VirtualMachine vm, + Account owner); String getVmNameFromVolumeId(long volumeId); diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/service/api/OrchestrationService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/service/api/OrchestrationService.java index 5a18b3cab9e..82c3dd14cf3 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/service/api/OrchestrationService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/service/api/OrchestrationService.java @@ -55,7 +55,7 @@ public interface OrchestrationService { * @param memory memory to allocate in bytes * @param computeTags tags for the compute * @param rootDiskTags tags for the root disk - * @param networks networks that this VM should join + * @param networkNicMap map networks to nic profiles that this VM should join * @param rootDiskSize size the root disk in case of templates. * @return VirtualMachineEntity */ @@ -65,7 +65,7 @@ public interface OrchestrationService { @QueryParam("host-name") String hostName, @QueryParam("display-name") String displayName, @QueryParam("hypervisor") String hypervisor, @QueryParam("cpu") int cpu, @QueryParam("speed") int speed, @QueryParam("ram") long memory, @QueryParam("disk-size") Long diskSize, @QueryParam("compute-tags") List computeTags, @QueryParam("root-disk-tags") List rootDiskTags, - @QueryParam("network-nic-map") Map networkNicMap, @QueryParam("deploymentplan") DeploymentPlan plan, + @QueryParam("network-nic-map") Map> networkNicMap, @QueryParam("deploymentplan") DeploymentPlan plan, @QueryParam("root-disk-size") Long rootDiskSize, @QueryParam("extra-dhcp-option-map") Map> extraDhcpOptionMap, @QueryParam("datadisktemplate-diskoffering-map") Map datadiskTemplateToDiskOfferingMap) throws InsufficientCapacityException; @@ -74,7 +74,7 @@ public interface OrchestrationService { @QueryParam("host-name") String hostName, @QueryParam("display-name") String displayName, @QueryParam("hypervisor") String hypervisor, @QueryParam("os") String os, @QueryParam("cpu") int cpu, @QueryParam("speed") int speed, @QueryParam("ram") long memory, @QueryParam("disk-size") Long diskSize, @QueryParam("compute-tags") List computeTags, @QueryParam("root-disk-tags") List rootDiskTags, - @QueryParam("network-nic-map") Map networkNicMap, @QueryParam("deploymentplan") DeploymentPlan plan, @QueryParam("extra-dhcp-option-map") Map> extraDhcpOptionMap) throws InsufficientCapacityException; + @QueryParam("network-nic-map") Map> networkNicMap, @QueryParam("deploymentplan") DeploymentPlan plan, @QueryParam("extra-dhcp-option-map") Map> extraDhcpOptionMap) throws InsufficientCapacityException; @POST NetworkEntity createNetwork(String id, String name, String domainName, String cidr, String gateway); diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStore.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStore.java index a399758217b..5546571ba6b 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStore.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStore.java @@ -23,7 +23,9 @@ import java.util.List; import org.apache.cloudstack.engine.subsystem.api.storage.disktype.DiskFormat; public interface PrimaryDataStore extends DataStore, PrimaryDataStoreInfo { - DataObject create(DataObject dataObject, boolean createEntryInTempSpoolRef); + DataObject create(DataObject dataObject, String configuration); + + DataObject create(DataObject dataObject, boolean createEntryInTempSpoolRef, String configuration); VolumeInfo getVolume(long id); @@ -31,7 +33,7 @@ public interface PrimaryDataStore extends DataStore, PrimaryDataStoreInfo { boolean exists(DataObject data); - TemplateInfo getTemplate(long templateId); + TemplateInfo getTemplate(long templateId, String configuration); SnapshotInfo getSnapshot(long snapshotId); diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateDataFactory.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateDataFactory.java index b213625efad..4d258f3b6d0 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateDataFactory.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateDataFactory.java @@ -27,7 +27,7 @@ public interface TemplateDataFactory { TemplateInfo getReadyTemplateOnImageStore(long templateId, Long zoneId); - TemplateInfo getTemplate(DataObject obj, DataStore store); + TemplateInfo getTemplate(DataObject obj, DataStore store, String configuration); TemplateInfo getTemplate(long templateId, DataStoreRole storeRole); @@ -40,4 +40,6 @@ public interface TemplateDataFactory { TemplateInfo getReadyBypassedTemplateOnPrimaryStore(long templateId, Long poolId, Long hostId); boolean isTemplateMarkedForDirectDownload(long templateId); + + TemplateInfo getTemplateOnPrimaryStorage(long templateId, DataStore store, String configuration); } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateInfo.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateInfo.java index 0f7cc6f9de5..1e4a1b7373a 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateInfo.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateInfo.java @@ -27,4 +27,8 @@ public interface TemplateInfo extends DataObject, VirtualMachineTemplate { String getInstallPath(); boolean isDirectDownload(); + + boolean isDeployAsIs(); + + String getDeployAsIsConfiguration(); } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java index f70a7813ae0..df13f951a44 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java @@ -18,6 +18,7 @@ */ package org.apache.cloudstack.engine.subsystem.api.storage; +import com.cloud.agent.api.to.DatadiskTO; import org.apache.cloudstack.framework.async.AsyncCallFuture; import org.apache.cloudstack.framework.async.AsyncCompletionCallback; import org.apache.cloudstack.storage.command.CommandResult; @@ -25,6 +26,8 @@ import org.apache.cloudstack.storage.command.CommandResult; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.storage.StoragePool; +import java.util.List; + public interface TemplateService { class TemplateApiResult extends CommandResult { @@ -47,7 +50,7 @@ public interface TemplateService { AsyncCallFuture createTemplateFromVolumeAsync(VolumeInfo volume, TemplateInfo template, DataStore store); - boolean createOvaDataDiskTemplates(TemplateInfo parentTemplate); + boolean createOvaDataDiskTemplates(TemplateInfo parentTemplate, boolean deployAsIs); AsyncCallFuture deleteTemplateAsync(TemplateInfo template); @@ -72,4 +75,6 @@ public interface TemplateService { void associateCrosszoneTemplatesToZone(long dcId); AsyncCallFuture createDatadiskTemplateAsync(TemplateInfo parentTemplate, TemplateInfo dataDiskTemplate, String path, String diskId, long fileSize, boolean bootable); + + List getTemplateDatadisksOnImageStore(TemplateInfo templateInfo, String configurationId); } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeInfo.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeInfo.java index f4a73810901..06bda51a092 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeInfo.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeInfo.java @@ -83,4 +83,8 @@ public interface VolumeInfo extends DataObject, Volume { boolean isDirectDownload(); void setDirectDownload(boolean directDownload); + + boolean isDeployAsIs(); + + String getDeployAsIsConfiguration(); } diff --git a/engine/api/src/main/java/org/apache/cloudstack/storage/image/datastore/ImageStoreEntity.java b/engine/api/src/main/java/org/apache/cloudstack/storage/image/datastore/ImageStoreEntity.java index 5a0be952b39..88d908d7874 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/storage/image/datastore/ImageStoreEntity.java +++ b/engine/api/src/main/java/org/apache/cloudstack/storage/image/datastore/ImageStoreEntity.java @@ -51,7 +51,7 @@ public interface ImageStoreEntity extends DataStore, ImageStore { void deleteExtractUrl(String installPath, String url, Upload.Type volume); - List getDataDiskTemplates(DataObject obj); + List getDataDiskTemplates(DataObject obj, String configurationId); Void createDataDiskTemplateAsync(TemplateInfo dataDiskTemplate, String path, String diskId, long fileSize, boolean bootable, AsyncCompletionCallback callback); } diff --git a/engine/components-api/src/main/java/com/cloud/template/TemplateManager.java b/engine/components-api/src/main/java/com/cloud/template/TemplateManager.java index 2dc6296fc51..35fed56ac8e 100644 --- a/engine/components-api/src/main/java/com/cloud/template/TemplateManager.java +++ b/engine/components-api/src/main/java/com/cloud/template/TemplateManager.java @@ -18,7 +18,9 @@ package com.cloud.template; import java.util.List; +import com.cloud.agent.api.to.DatadiskTO; import com.cloud.deploy.DeployDestination; +import com.cloud.storage.DataStoreRole; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; import org.apache.cloudstack.framework.config.ConfigKey; @@ -133,4 +135,5 @@ public interface TemplateManager { public static final String MESSAGE_REGISTER_PUBLIC_TEMPLATE_EVENT = "Message.RegisterPublicTemplate.Event"; public static final String MESSAGE_RESET_TEMPLATE_PERMISSION_EVENT = "Message.ResetTemplatePermission.Event"; + List getTemplateDisksOnImageStore(Long templateId, DataStoreRole role, String configurationId); } diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index ffd5878bbf6..fb4fadd9500 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -414,6 +414,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac final LinkedHashMap> auxiliaryNetworks, final DeploymentPlan plan, final HypervisorType hyperType, final Map> extraDhcpOptions, final Map datadiskTemplateToDiskOfferingMap) throws InsufficientCapacityException { + s_logger.info(String.format("allocating virtual machine from template:%s with hostname:%s and %d networks", template.getUuid(), vmInstanceName, auxiliaryNetworks.size())); + final VMInstanceVO vm = _vmDao.findVMByInstanceName(vmInstanceName); final Account owner = _entityMgr.findById(Account.class, vm.getAccountId()); @@ -455,7 +457,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } else if (template.getFormat() == ImageFormat.BAREMETAL) { // Do nothing } else { - volumeMgr.allocateTemplatedVolume(Type.ROOT, "ROOT-" + vmFinal.getId(), rootDiskOfferingInfo.getDiskOffering(), rootDiskOfferingInfo.getSize(), + volumeMgr.allocateTemplatedVolumes(Type.ROOT, "ROOT-" + vmFinal.getId(), rootDiskOfferingInfo.getDiskOffering(), rootDiskOfferingInfo.getSize(), rootDiskOfferingInfo.getMinIops(), rootDiskOfferingInfo.getMaxIops(), template, vmFinal, owner); } @@ -1104,7 +1106,9 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac if (dest != null) { avoids.addHost(dest.getHost().getId()); - journal.record("Deployment found ", vmProfile, dest); + if (!template.isDeployAsIs()) { + journal.record("Deployment found ", vmProfile, dest); + } } long destHostId = dest.getHost().getId(); @@ -1468,6 +1472,11 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac if (disk.getType() != Volume.Type.ISO) { final VolumeObjectTO vol = (VolumeObjectTO)disk.getData(); final VolumeVO volume = _volsDao.findById(vol.getId()); + if (vmSpec.getDeployAsIsInfo() != null && vmSpec.getDeployAsIsInfo().isDeployAsIs() + && StringUtils.isNotBlank(vol.getPath())) { + volume.setPath(vol.getPath()); + _volsDao.update(volume.getId(), volume); + } // Use getPath() from VolumeVO to get a fresh copy of what's in the DB. // Before doing this, in a certain situation, getPath() from VolumeObjectTO diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/CloudOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/CloudOrchestrator.java index 91e9b6f57bd..c6f8e418642 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/CloudOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/CloudOrchestrator.java @@ -20,7 +20,6 @@ package org.apache.cloudstack.engine.orchestration; import java.net.URL; import java.util.ArrayList; -import java.util.Arrays; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -156,7 +155,7 @@ public class CloudOrchestrator implements OrchestrationService { @Override public VirtualMachineEntity createVirtualMachine(String id, String owner, String templateId, String hostName, String displayName, String hypervisor, int cpu, - int speed, long memory, Long diskSize, List computeTags, List rootDiskTags, Map networkNicMap, DeploymentPlan plan, + int speed, long memory, Long diskSize, List computeTags, List rootDiskTags, Map> networkNicMap, DeploymentPlan plan, Long rootDiskSize, Map> extraDhcpOptionMap, Map dataDiskTemplateToDiskOfferingMap) throws InsufficientCapacityException { // VirtualMachineEntityImpl vmEntity = new VirtualMachineEntityImpl(id, owner, hostName, displayName, cpu, speed, memory, computeTags, rootDiskTags, networks, @@ -166,7 +165,7 @@ public class CloudOrchestrator implements OrchestrationService { for (String uuid : networkNicMap.keySet()) { NetworkVO network = _networkDao.findByUuid(uuid); if(network != null){ - networkIpMap.put(network, new ArrayList(Arrays.asList(networkNicMap.get(uuid)))); + networkIpMap.put(network, networkNicMap.get(uuid)); } } @@ -255,7 +254,7 @@ public class CloudOrchestrator implements OrchestrationService { @Override public VirtualMachineEntity createVirtualMachineFromScratch(String id, String owner, String isoId, String hostName, String displayName, String hypervisor, String os, - int cpu, int speed, long memory, Long diskSize, List computeTags, List rootDiskTags, Map networkNicMap, DeploymentPlan plan, Map> extraDhcpOptionMap) + int cpu, int speed, long memory, Long diskSize, List computeTags, List rootDiskTags, Map> networkNicMap, DeploymentPlan plan, Map> extraDhcpOptionMap) throws InsufficientCapacityException { // VirtualMachineEntityImpl vmEntity = new VirtualMachineEntityImpl(id, owner, hostName, displayName, cpu, speed, memory, computeTags, rootDiskTags, networks, vmEntityManager); @@ -307,7 +306,7 @@ public class CloudOrchestrator implements OrchestrationService { for (String uuid : networkNicMap.keySet()) { NetworkVO network = _networkDao.findByUuid(uuid); if(network != null){ - networkIpMap.put(network, new ArrayList(Arrays.asList(networkNicMap.get(uuid)))); + networkIpMap.put(network, networkNicMap.get(uuid)); } } diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java index e12dca04e31..5b635bd2255 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java @@ -16,6 +16,8 @@ // under the License. package org.apache.cloudstack.engine.orchestration; +import static org.apache.commons.lang.StringUtils.isNotBlank; + import java.net.URI; import java.util.ArrayList; import java.util.Arrays; @@ -38,12 +40,9 @@ import java.util.stream.Collectors; import javax.inject.Inject; import javax.naming.ConfigurationException; -import com.cloud.event.EventTypes; -import com.cloud.event.UsageEventUtils; -import com.cloud.network.dao.NetworkDetailVO; -import com.cloud.network.dao.NetworkDetailsDao; import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.net.NetworkPrerequisiteTO; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.cloud.entity.api.db.VMNetworkMapVO; import org.apache.cloudstack.engine.cloud.entity.api.db.dao.VMNetworkMapDao; @@ -88,6 +87,8 @@ import com.cloud.deploy.DataCenterDeployment; import com.cloud.deploy.DeployDestination; import com.cloud.deploy.DeploymentPlan; import com.cloud.domain.Domain; +import com.cloud.event.EventTypes; +import com.cloud.event.UsageEventUtils; import com.cloud.exception.ConcurrentOperationException; import com.cloud.exception.ConnectionException; import com.cloud.exception.InsufficientAddressCapacityException; @@ -129,6 +130,8 @@ import com.cloud.network.dao.IPAddressVO; import com.cloud.network.dao.NetworkAccountDao; import com.cloud.network.dao.NetworkAccountVO; import com.cloud.network.dao.NetworkDao; +import com.cloud.network.dao.NetworkDetailVO; +import com.cloud.network.dao.NetworkDetailsDao; import com.cloud.network.dao.NetworkDomainDao; import com.cloud.network.dao.NetworkDomainVO; import com.cloud.network.dao.NetworkServiceMapDao; @@ -178,6 +181,7 @@ import com.cloud.offerings.NetworkOfferingVO; import com.cloud.offerings.dao.NetworkOfferingDao; import com.cloud.offerings.dao.NetworkOfferingDetailsDao; import com.cloud.offerings.dao.NetworkOfferingServiceMapDao; +import com.cloud.storage.dao.VMTemplateDetailsDao; import com.cloud.user.Account; import com.cloud.user.ResourceLimitService; import com.cloud.user.User; @@ -231,8 +235,6 @@ import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.VMInstanceDao; import com.google.common.base.Strings; -import static org.apache.commons.lang.StringUtils.isNotBlank; - /** * NetworkManagerImpl implements NetworkManager. */ @@ -301,6 +303,8 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra VpcVirtualNetworkApplianceService _routerService; @Inject UserVmManager _userVmMgr; + @Inject + VMTemplateDetailsDao templateDetailsDao; List networkGurus; @@ -645,8 +649,8 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra @Override @DB public List setupNetwork(final Account owner, final NetworkOffering offering, final Network predefined, final DeploymentPlan plan, final String name, - final String displayText, final boolean errorIfAlreadySetup, final Long domainId, final ACLType aclType, final Boolean subdomainAccess, final Long vpcId, - final Boolean isDisplayNetworkEnabled) throws ConcurrentOperationException { + final String displayText, final boolean errorIfAlreadySetup, final Long domainId, final ACLType aclType, final Boolean subdomainAccess, final Long vpcId, + final Boolean isDisplayNetworkEnabled) throws ConcurrentOperationException { final Account locked = _accountDao.acquireInLockTable(owner.getId()); if (locked == null) { @@ -656,8 +660,8 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra try { if (predefined == null || offering.getTrafficType() != TrafficType.Guest && predefined.getCidr() == null && predefined.getBroadcastUri() == null && !(predefined - .getBroadcastDomainType() == BroadcastDomainType.Vlan || predefined.getBroadcastDomainType() == BroadcastDomainType.Lswitch || predefined - .getBroadcastDomainType() == BroadcastDomainType.Vxlan)) { + .getBroadcastDomainType() == BroadcastDomainType.Vlan || predefined.getBroadcastDomainType() == BroadcastDomainType.Lswitch || predefined + .getBroadcastDomainType() == BroadcastDomainType.Vxlan)) { final List configs = _networksDao.listBy(owner.getId(), offering.getId(), plan.getDataCenterId()); if (configs.size() > 0) { if (s_logger.isDebugEnabled()) { @@ -747,12 +751,129 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra @Override @DB public void allocate(final VirtualMachineProfile vm, final LinkedHashMap> networks, final Map> extraDhcpOptions) throws InsufficientCapacityException, - ConcurrentOperationException { + ConcurrentOperationException { Transaction.execute(new TransactionCallbackWithExceptionNoReturn() { @Override public void doInTransactionWithoutResult(final TransactionStatus status) throws InsufficientCapacityException { + if (s_logger.isTraceEnabled()) { + s_logger.trace(String.format("allocating networks for %s(template %s); %d networks",vm.getInstanceName(), vm.getTemplate().getUuid(), networks.size())); + } int deviceId = 0; + int size; + size = determineNumberOfNicsRequired(); + + final boolean[] deviceIds = new boolean[size]; + Arrays.fill(deviceIds, false); + + List> profilesList = getOrderedNetworkNicProfileMapping(networks); + final List nics = new ArrayList(size); + NicProfile defaultNic = null; + Network nextNetwork = null; + for (Pair networkNicPair : profilesList) { + nextNetwork = networkNicPair.first(); + Pair newDeviceInfo = addRequestedNicToNicListWithDeviceNumberAndRetrieveDefaultDevice(networkNicPair.second(), deviceIds, deviceId, nextNetwork, nics, defaultNic); + defaultNic = newDeviceInfo.first(); + deviceId = newDeviceInfo.second(); + } + createExtraNics(size, nics, nextNetwork); + + if (nics.size() == 1) { + nics.get(0).setDefaultNic(true); + } + } + + /** + * private transaction method to check and add devices to the nic list and update the info + */ + Pair addRequestedNicToNicListWithDeviceNumberAndRetrieveDefaultDevice(NicProfile requested, boolean[] deviceIds, int deviceId, Network nextNetwork, List nics, NicProfile defaultNic) + throws InsufficientAddressCapacityException, InsufficientVirtualNetworkCapacityException { + Pair rc = new Pair<>(null,null); + Boolean isDefaultNic = false; + if (vm != null && requested != null && requested.isDefaultNic()) { + isDefaultNic = true; + } + + while (deviceIds[deviceId] && deviceId < deviceIds.length) { + deviceId++; + } + + final Pair vmNicPair = allocateNic(requested, nextNetwork, isDefaultNic, deviceId, vm); + NicProfile vmNic = null; + if (vmNicPair != null) { + vmNic = vmNicPair.first(); + if (vmNic == null) { + return rc; + } + deviceId = vmNicPair.second(); + } + + final int devId = vmNic.getDeviceId(); + if (devId >= deviceIds.length) { + throw new IllegalArgumentException("Device id for nic is too large: " + vmNic); + } + if (deviceIds[devId]) { + throw new IllegalArgumentException("Conflicting device id for two different nics: " + vmNic); + } + + deviceIds[devId] = true; + + if (vmNic.isDefaultNic()) { + if (defaultNic != null) { + throw new IllegalArgumentException("You cannot specify two nics as default nics: nic 1 = " + defaultNic + "; nic 2 = " + vmNic); + } + defaultNic = vmNic; + } + + nics.add(vmNic); + vm.addNic(vmNic); + saveExtraDhcpOptions(nextNetwork.getUuid(), vmNic.getId(), extraDhcpOptions); + rc.first(defaultNic); + rc.second(deviceId); + return rc; + } + + /** + * private transaction method to get oredered list of Network and NicProfile pair + * @return ordered list of Network and NicProfile pair + * @param networks the map od networks to nic profiles list + */ + private List> getOrderedNetworkNicProfileMapping(final LinkedHashMap> networks) { + List> profilesList = new ArrayList<>(); + for (final Map.Entry> network : networks.entrySet()) { + List requestedProfiles = network.getValue(); + if (requestedProfiles == null) { + requestedProfiles = new ArrayList(); + } + if (requestedProfiles.isEmpty()) { + requestedProfiles.add(null); + } + for (final NicProfile requested : requestedProfiles) { + profilesList.add(new Pair(network.getKey(), requested)); + } + } + profilesList.sort(new Comparator>() { + @Override + public int compare(Pair pair1, Pair pair2) { + int profile1Order = Integer.MAX_VALUE; + int profile2Order = Integer.MAX_VALUE; + if (pair1 != null && pair1.second() != null && pair1.second().getOrderIndex() != null) { + profile1Order = pair1.second().getOrderIndex(); + } + if (pair2 != null && pair2.second() != null && pair2.second().getOrderIndex() != null) { + profile2Order = pair2.second().getOrderIndex(); + } + return profile1Order - profile2Order; + } + }); + return profilesList; + } + + /** + * private transaction method to run over the objects and determine nic requirements + * @return the total numer of nics required + */ + private int determineNumberOfNicsRequired() { int size = 0; for (final Network ntwk : networks.keySet()) { final List profiles = networks.get(ntwk); @@ -763,71 +884,35 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra } } - final boolean[] deviceIds = new boolean[size]; - Arrays.fill(deviceIds, false); - - final List nics = new ArrayList(size); - NicProfile defaultNic = null; - - for (final Map.Entry> network : networks.entrySet()) { - final Network config = network.getKey(); - List requestedProfiles = network.getValue(); - if (requestedProfiles == null) { - requestedProfiles = new ArrayList(); - } - if (requestedProfiles.isEmpty()) { - requestedProfiles.add(null); - } - - for (final NicProfile requested : requestedProfiles) { - Boolean isDefaultNic = false; - if (vm != null && requested != null && requested.isDefaultNic()) { - isDefaultNic = true; - } - - while (deviceIds[deviceId] && deviceId < deviceIds.length) { - deviceId++; - } - - final Pair vmNicPair = allocateNic(requested, config, isDefaultNic, deviceId, vm); - NicProfile vmNic = null; - if (vmNicPair != null) { - vmNic = vmNicPair.first(); - if (vmNic == null) { - continue; - } - deviceId = vmNicPair.second(); - } - - final int devId = vmNic.getDeviceId(); - if (devId >= deviceIds.length) { - throw new IllegalArgumentException("Device id for nic is too large: " + vmNic); - } - if (deviceIds[devId]) { - throw new IllegalArgumentException("Conflicting device id for two different nics: " + vmNic); - } - - deviceIds[devId] = true; - - if (vmNic.isDefaultNic()) { - if (defaultNic != null) { - throw new IllegalArgumentException("You cannot specify two nics as default nics: nic 1 = " + defaultNic + "; nic 2 = " + vmNic); - } - defaultNic = vmNic; - } - - nics.add(vmNic); - vm.addNic(vmNic); - saveExtraDhcpOptions(config.getUuid(), vmNic.getId(), extraDhcpOptions); - } + List netprereqs = templateDetailsDao.listNetworkRequirementsByTemplateId(vm.getTemplate().getId()); + if (size < netprereqs.size()) { + size = netprereqs.size(); } + return size; + } + + /** + * private transaction method to add nics as required + * @param size the number needed + * @param nics the list of nics present + * @param finalNetwork the network to add the nics to + * @throws InsufficientVirtualNetworkCapacityException great + * @throws InsufficientAddressCapacityException also magnificent, as the name sugests + */ + private void createExtraNics(int size, List nics, Network finalNetwork) throws InsufficientVirtualNetworkCapacityException, InsufficientAddressCapacityException { if (nics.size() != size) { s_logger.warn("Number of nics " + nics.size() + " doesn't match number of requested nics " + size); - throw new CloudRuntimeException("Number of nics " + nics.size() + " doesn't match number of requested networks " + size); - } - - if (nics.size() == 1) { - nics.get(0).setDefaultNic(true); + if (nics.size() > size) { + throw new CloudRuntimeException("Number of nics " + nics.size() + " doesn't match number of requested networks " + size); + } else { + if (finalNetwork == null) { + throw new CloudRuntimeException(String.format("can not assign network to %d remaining required NICs", size - nics.size())); + } + // create extra + for ( int extraNicNum = nics.size() ; extraNicNum < size; extraNicNum ++) { + final Pair vmNicPair = allocateNic(new NicProfile(), finalNetwork, false, extraNicNum, vm); + } + } } } }); @@ -1085,7 +1170,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra } Pair implementNetwork(final long networkId, final DeployDestination dest, final ReservationContext context, final boolean isRouter) throws ConcurrentOperationException, - ResourceUnavailableException, InsufficientCapacityException { + ResourceUnavailableException, InsufficientCapacityException { Pair implemented = null; if (!isRouter) { implemented = implementNetwork(networkId, dest, context); @@ -1105,7 +1190,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra @Override @DB public Pair implementNetwork(final long networkId, final DeployDestination dest, final ReservationContext context) throws ConcurrentOperationException, - ResourceUnavailableException, InsufficientCapacityException { + ResourceUnavailableException, InsufficientCapacityException { final Pair implemented = new Pair(null, null); NetworkVO network = _networksDao.findById(networkId); @@ -1396,7 +1481,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra } protected boolean prepareElement(final NetworkElement element, final Network network, final NicProfile profile, final VirtualMachineProfile vmProfile, final DeployDestination dest, - final ReservationContext context) throws InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException { + final ReservationContext context) throws InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException { element.prepare(network, profile, vmProfile, dest, context); if (vmProfile.getType() == Type.User && element.getProvider() != null) { if (_networkModel.areServicesSupportedInNetwork(network.getId(), Service.Dhcp) @@ -1582,11 +1667,11 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra if (element instanceof RedundantResource) { resourceCount= ((RedundantResource) element).getResourceCount(network); break; - } } } - return resourceCount; } + return resourceCount; + } @Override public void configureExtraDhcpOptions(Network network, long nicId, Map extraDhcpOptions) { @@ -1628,12 +1713,12 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra _networkModel.getNetworkTag(vm.getHypervisorType(), network)); for (final NetworkElement element : networkElements) { if (_networkModel.areServicesSupportedInNetwork(network.getId(), Service.UserData) && element instanceof UserDataServiceProvider) { - if (element instanceof ConfigDriveNetworkElement && !migrationSuccessful || element instanceof VirtualRouterElement && migrationSuccessful) { - final UserDataServiceProvider sp = (UserDataServiceProvider) element; - if (!sp.saveHypervisorHostname(profile, network, vm, dest)) { - throw new CloudRuntimeException("Failed to Add hypervisor hostname"); - } + if (element instanceof ConfigDriveNetworkElement && !migrationSuccessful || element instanceof VirtualRouterElement && migrationSuccessful) { + final UserDataServiceProvider sp = (UserDataServiceProvider) element; + if (!sp.saveHypervisorHostname(profile, network, vm, dest)) { + throw new CloudRuntimeException("Failed to Add hypervisor hostname"); } + } } } } @@ -1661,7 +1746,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra @Override public void prepare(final VirtualMachineProfile vmProfile, final DeployDestination dest, final ReservationContext context) throws InsufficientCapacityException, ConcurrentOperationException, - ResourceUnavailableException { + ResourceUnavailableException { final List nics = _nicDao.listByVmId(vmProfile.getId()); // we have to implement default nics first - to ensure that default network elements start up first in multiple @@ -2246,9 +2331,9 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra @DB private Network createGuestNetwork(final long networkOfferingId, final String name, final String displayText, final String gateway, final String cidr, String vlanId, - boolean bypassVlanOverlapCheck, String networkDomain, final Account owner, final Long domainId, final PhysicalNetwork pNtwk, - final long zoneId, final ACLType aclType, Boolean subdomainAccess, final Long vpcId, final String ip6Gateway, final String ip6Cidr, - final Boolean isDisplayNetworkEnabled, final String isolatedPvlan, Network.PVlanType isolatedPvlanType, String externalId, final Boolean isPrivateNetwork) throws ConcurrentOperationException, InsufficientCapacityException, ResourceAllocationException { + boolean bypassVlanOverlapCheck, String networkDomain, final Account owner, final Long domainId, final PhysicalNetwork pNtwk, + final long zoneId, final ACLType aclType, Boolean subdomainAccess, final Long vpcId, final String ip6Gateway, final String ip6Cidr, + final Boolean isDisplayNetworkEnabled, final String isolatedPvlan, Network.PVlanType isolatedPvlanType, String externalId, final Boolean isPrivateNetwork) throws ConcurrentOperationException, InsufficientCapacityException, ResourceAllocationException { final NetworkOfferingVO ntwkOff = _networkOfferingDao.findById(networkOfferingId); final DataCenterVO zone = _dcDao.findById(zoneId); @@ -2364,12 +2449,12 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra URI secondaryUri = isNotBlank(isolatedPvlan) ? BroadcastDomainType.fromString(isolatedPvlan) : null; //don't allow to specify vlan tag used by physical network for dynamic vlan allocation if (!(bypassVlanOverlapCheck && ntwkOff.getGuestType() == GuestType.Shared) && _dcDao.findVnet(zoneId, pNtwk.getId(), BroadcastDomainType.getValue(uri)).size() > 0) { - throw new InvalidParameterValueException("The VLAN tag " + vlanId + " is already being used for dynamic vlan allocation for the guest network in zone " + throw new InvalidParameterValueException("The VLAN tag to use for new guest network, " + vlanId + " is already being used for dynamic vlan allocation for the guest network in zone " + zone.getName()); } if (secondaryUri != null && !(bypassVlanOverlapCheck && ntwkOff.getGuestType() == GuestType.Shared) && _dcDao.findVnet(zoneId, pNtwk.getId(), BroadcastDomainType.getValue(secondaryUri)).size() > 0) { - throw new InvalidParameterValueException("The VLAN tag " + isolatedPvlan + " is already being used for dynamic vlan allocation for the guest network in zone " + throw new InvalidParameterValueException("The VLAN tag for isolated PVLAN " + isolatedPvlan + " is already being used for dynamic vlan allocation for the guest network in zone " + zone.getName()); } if (! UuidUtils.validateUUID(vlanId)){ @@ -2605,7 +2690,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra return bypassVlanOverlapCheck && (ntwkOff.getGuestType() != GuestType.Isolated || isPrivateNetwork); } - /** + /** * Checks for L2 network offering services. Only 2 cases allowed: * - No services * - User Data service only, provided by ConfigDrive @@ -3033,7 +3118,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra @Override public boolean startNetwork(final long networkId, final DeployDestination dest, final ReservationContext context) throws ConcurrentOperationException, ResourceUnavailableException, - InsufficientCapacityException { + InsufficientCapacityException { // Check if network exists final NetworkVO network = _networksDao.findById(networkId); @@ -3056,7 +3141,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra @Override public boolean restartNetwork(final Long networkId, final Account callerAccount, final User callerUser, final boolean cleanup) throws ConcurrentOperationException, ResourceUnavailableException, - InsufficientCapacityException { + InsufficientCapacityException { boolean status = true; boolean restartRequired = false; final NetworkVO network = _networksDao.findById(networkId); @@ -3308,10 +3393,10 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra final NetworkOfferingVO networkOffering = _networkOfferingDao.findById(networkOfferingId); if (networkOffering.getGuestType() == Network.GuestType.Shared && (_networkModel.areServicesSupportedByNetworkOffering(networkOfferingId, Service.SourceNat) - || _networkModel.areServicesSupportedByNetworkOffering(networkOfferingId, Service.StaticNat) - || _networkModel.areServicesSupportedByNetworkOffering(networkOfferingId, Service.Firewall) - || _networkModel.areServicesSupportedByNetworkOffering(networkOfferingId, Service.PortForwarding) || _networkModel.areServicesSupportedByNetworkOffering( - networkOfferingId, Service.Lb))) { + || _networkModel.areServicesSupportedByNetworkOffering(networkOfferingId, Service.StaticNat) + || _networkModel.areServicesSupportedByNetworkOffering(networkOfferingId, Service.Firewall) + || _networkModel.areServicesSupportedByNetworkOffering(networkOfferingId, Service.PortForwarding) || _networkModel.areServicesSupportedByNetworkOffering( + networkOfferingId, Service.Lb))) { return true; } return false; @@ -4183,4 +4268,4 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra GuestDomainSuffix, NetworkThrottlingRate, MinVRVersion, PromiscuousMode, MacAddressChanges, ForgedTransmits, RollingRestartEnabled}; } -} +} \ No newline at end of file diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java index 958771c561d..a51aae7c8f4 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java @@ -19,6 +19,7 @@ package org.apache.cloudstack.engine.orchestration; import java.util.ArrayList; +import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -26,10 +27,18 @@ import java.util.Map; import java.util.Set; import java.util.UUID; import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.agent.api.to.DatadiskTO; +import com.cloud.storage.VolumeDetailVO; +import com.cloud.storage.dao.VMTemplateDetailsDao; +import com.cloud.utils.StringUtils; +import com.cloud.vm.UserVmDetailVO; +import com.cloud.vm.VmDetailConstants; +import com.cloud.vm.dao.UserVmDetailsDao; import org.apache.cloudstack.api.command.admin.vm.MigrateVMCmd; import org.apache.cloudstack.api.command.admin.volume.MigrateVolumeCmdByAdmin; import org.apache.cloudstack.api.command.user.volume.MigrateVolumeCmd; @@ -51,6 +60,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.StoragePoolAllocator; import org.apache.cloudstack.engine.subsystem.api.storage.StorageStrategyFactory; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService; @@ -68,6 +78,7 @@ import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; +import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; import com.cloud.agent.api.to.DataTO; @@ -141,6 +152,8 @@ import com.cloud.vm.dao.UserVmCloneSettingDao; import com.cloud.vm.dao.UserVmDao; import static com.cloud.utils.NumbersUtil.toHumanReadableSize; +import static com.cloud.storage.resource.StorageProcessor.REQUEST_TEMPLATE_RELOAD; + public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrationService, Configurable { public enum UserVmCloneType { @@ -197,6 +210,12 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati protected UserVmCloneSettingDao _vmCloneSettingDao; @Inject StorageStrategyFactory _storageStrategyFactory; + @Inject + VMTemplateDetailsDao templateDetailsDao; + @Inject + TemplateService templateService; + @Inject + UserVmDetailsDao userVmDetailsDao; private final StateMachine2 _volStateMachine; protected List _storagePoolAllocators; @@ -512,7 +531,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati @DB public VolumeInfo copyVolumeFromSecToPrimary(VolumeInfo volume, VirtualMachine vm, VirtualMachineTemplate template, DataCenter dc, Pod pod, Long clusterId, ServiceOffering offering, - DiskOffering diskOffering, List avoids, long size, HypervisorType hyperType) throws NoTransitionException { + DiskOffering diskOffering, List avoids, long size, HypervisorType hyperType) throws NoTransitionException { final HashSet avoidPools = new HashSet(avoids); DiskProfile dskCh = createDiskCharacteristics(volume, template, dc, diskOffering); @@ -545,7 +564,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati @DB public VolumeInfo createVolume(VolumeInfo volume, VirtualMachine vm, VirtualMachineTemplate template, DataCenter dc, Pod pod, Long clusterId, ServiceOffering offering, DiskOffering diskOffering, - List avoids, long size, HypervisorType hyperType) { + List avoids, long size, HypervisorType hyperType) { // update the volume's hv_ss_reserve (hypervisor snapshot reserve) from a disk offering (used for managed storage) volume = volService.updateHypervisorSnapshotReserveForVolume(diskOffering, volume.getId(), hyperType); @@ -591,7 +610,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati try { VolumeApiResult result = future.get(); if (result.isFailed()) { - if (result.getResult().contains("request template reload") && (i == 0)) { + if (result.getResult().contains(REQUEST_TEMPLATE_RELOAD) && (i == 0)) { s_logger.debug("Retry template re-deploy for vmware"); continue; } else { @@ -687,7 +706,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati @Override public DiskProfile allocateRawVolume(Type type, String name, DiskOffering offering, Long size, Long minIops, Long maxIops, VirtualMachine vm, VirtualMachineTemplate template, Account owner, - Long deviceId) { + Long deviceId) { if (size == null) { size = offering.getDiskSize(); } else { @@ -734,19 +753,23 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati return toDiskProfile(vol, offering); } - @Override - public DiskProfile allocateTemplatedVolume(Type type, String name, DiskOffering offering, Long rootDisksize, Long minIops, Long maxIops, VirtualMachineTemplate template, VirtualMachine vm, - Account owner) { + private DiskProfile allocateTemplatedVolume(Type type, String name, DiskOffering offering, Long rootDisksize, Long minIops, Long maxIops, VirtualMachineTemplate template, VirtualMachine vm, + Account owner, long deviceId, String configurationId) { assert (template.getFormat() != ImageFormat.ISO) : "ISO is not a template really...."; Long size = _tmpltMgr.getTemplateSize(template.getId(), vm.getDataCenterId()); if (rootDisksize != null) { - rootDisksize = rootDisksize * 1024 * 1024 * 1024; - if (rootDisksize > size) { - s_logger.debug("Using root disk size of " + toHumanReadableSize(rootDisksize) + " Bytes for volume " + name); + if (template.isDeployAsIs()) { + // Volume size specified from template deploy-as-is size = rootDisksize; } else { - s_logger.debug("Using root disk size of " + toHumanReadableSize(size) + " Bytes for volume " + name + "since specified root disk size of " + toHumanReadableSize(rootDisksize) + " Bytes is smaller than template"); + rootDisksize = rootDisksize * 1024 * 1024 * 1024; + if (rootDisksize > size) { + s_logger.debug("Using root disk size of " + toHumanReadableSize(rootDisksize) + " Bytes for volume " + name); + size = rootDisksize; + } else { + s_logger.debug("Using root disk size of " + toHumanReadableSize(rootDisksize) + " Bytes for volume " + name + "since specified root disk size of " + rootDisksize + " Bytes is smaller than template"); + } } } @@ -760,13 +783,9 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati } vol.setTemplateId(template.getId()); - if (type.equals(Type.ROOT)) { - vol.setDeviceId(0l); - if (!vm.getType().equals(VirtualMachine.Type.User)) { - vol.setRecreatable(true); - } - } else { - vol.setDeviceId(1l); + vol.setDeviceId(deviceId); + if (type.equals(Type.ROOT) && !vm.getType().equals(VirtualMachine.Type.User)) { + vol.setRecreatable(true); } if (vm.getType() == VirtualMachine.Type.User) { @@ -776,6 +795,11 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati vol = _volsDao.persist(vol); + if (StringUtils.isNotBlank(configurationId)) { + VolumeDetailVO deployConfigurationDetail = new VolumeDetailVO(vol.getId(), VmDetailConstants.DEPLOY_AS_IS_CONFIGURATION, configurationId, false); + _volDetailDao.persist(deployConfigurationDetail); + } + // Create event and update resource count for volumes if vm is a user vm if (vm.getType() == VirtualMachine.Type.User) { @@ -794,6 +818,51 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati return toDiskProfile(vol, offering); } + @Override + public List allocateTemplatedVolumes(Type type, String name, DiskOffering offering, Long rootDisksize, Long minIops, Long maxIops, VirtualMachineTemplate template, VirtualMachine vm, + Account owner) { + int volumesNumber = 1; + List templateAsIsDisks = null; + String configurationId = null; + if (template.isDeployAsIs()) { + UserVmDetailVO configurationDetail = userVmDetailsDao.findDetail(vm.getId(), VmDetailConstants.DEPLOY_AS_IS_CONFIGURATION); + if (configurationDetail != null) { + configurationId = configurationDetail.getValue(); + } + templateAsIsDisks = _tmpltMgr.getTemplateDisksOnImageStore(template.getId(), DataStoreRole.Image, configurationId); + if (CollectionUtils.isNotEmpty(templateAsIsDisks)) { + templateAsIsDisks = templateAsIsDisks.stream() + .filter(x -> !x.isIso()) + .sorted(Comparator.comparing(DatadiskTO::getDiskNumber)) + .collect(Collectors.toList()); + } + volumesNumber = templateAsIsDisks.size(); + } + + if (volumesNumber < 1) { + throw new CloudRuntimeException("Unable to create any volume from template " + template.getName()); + } + + List profiles = new ArrayList<>(); + + for (int number = 0; number < volumesNumber; number++) { + String volumeName = name; + Long volumeSize = rootDisksize; + long deviceId = type.equals(Type.ROOT) ? 0L : 1L; + if (template.isDeployAsIs()) { + int volumeNameSuffix = templateAsIsDisks.get(number).getDiskNumber(); + volumeName = String.format("%s-%d", volumeName, volumeNameSuffix); + volumeSize = templateAsIsDisks.get(number).getVirtualSize(); + deviceId = templateAsIsDisks.get(number).getDiskNumber(); + } + s_logger.info(String.format("adding disk object %s to %s", volumeName, vm.getInstanceName())); + DiskProfile diskProfile = allocateTemplatedVolume(type, volumeName, offering, volumeSize, minIops, maxIops, + template, vm, owner, deviceId, configurationId); + profiles.add(diskProfile); + } + return profiles; + } + private ImageFormat getSupportedImageFormatForCluster(HypervisorType hyperType) { if (hyperType == HypervisorType.XenServer) { return ImageFormat.VHD; @@ -824,7 +893,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati } private VolumeInfo copyVolume(StoragePool rootDiskPool, VolumeInfo volume, VirtualMachine vm, VirtualMachineTemplate rootDiskTmplt, DataCenter dcVO, Pod pod, DiskOffering diskVO, - ServiceOffering svo, HypervisorType rootDiskHyperType) throws NoTransitionException { + ServiceOffering svo, HypervisorType rootDiskHyperType) throws NoTransitionException { if (!isSupportedImageFormatForCluster(volume, rootDiskHyperType)) { throw new InvalidParameterValueException("Failed to attach volume to VM since volumes format " + volume.getFormat().getFileExtension() + " is not compatible with the vm hypervisor type"); @@ -1356,7 +1425,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati try { result = future.get(); if (result.isFailed()) { - if (result.getResult().contains("request template reload") && (i == 0)) { + if (result.getResult().contains(REQUEST_TEMPLATE_RELOAD) && (i == 0)) { s_logger.debug("Retry template re-deploy for vmware"); continue; } else { @@ -1732,4 +1801,4 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati } }); } -} +} \ No newline at end of file diff --git a/engine/schema/src/main/java/com/cloud/storage/TemplateOVFPropertyVO.java b/engine/schema/src/main/java/com/cloud/storage/TemplateOVFPropertyVO.java deleted file mode 100644 index 425b1f22e45..00000000000 --- a/engine/schema/src/main/java/com/cloud/storage/TemplateOVFPropertyVO.java +++ /dev/null @@ -1,167 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloud.storage; - -import com.cloud.agent.api.storage.OVFProperty; - -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.GeneratedValue; -import javax.persistence.GenerationType; -import javax.persistence.Id; -import javax.persistence.Table; - -@Entity -@Table(name = "template_ovf_properties") -public class TemplateOVFPropertyVO implements OVFProperty { - - @Id - @GeneratedValue(strategy = GenerationType.IDENTITY) - @Column(name = "id") - private long id; - - @Column(name = "template_id") - private Long templateId; - - @Column(name = "key") - private String key; - - @Column(name = "type") - private String type; - - @Column(name = "value") - private String value; - - @Column(name = "qualifiers") - private String qualifiers; - - @Column(name = "password") - private Boolean password; - - @Column(name = "user_configurable") - private Boolean userConfigurable; - - @Column(name = "label") - private String label; - - @Column(name = "description") - private String description; - - public TemplateOVFPropertyVO() { - } - - public TemplateOVFPropertyVO(Long templateId, String key, String type, String value, String qualifiers, - Boolean userConfigurable, String label, String description, Boolean password) { - this.templateId = templateId; - this.key = key; - this.type = type; - this.value = value; - this.qualifiers = qualifiers; - this.userConfigurable = userConfigurable; - this.label = label; - this.description = description; - this.password = password; - } - - public long getId() { - return id; - } - - public void setId(long id) { - this.id = id; - } - - public Long getTemplateId() { - return templateId; - } - - public void setTemplateId(Long templateId) { - this.templateId = templateId; - } - - public String getKey() { - return key; - } - - public void setKey(String key) { - this.key = key; - } - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } - - public String getValue() { - return value; - } - - public void setValue(String value) { - this.value = value; - } - - public String getQualifiers() { - return qualifiers; - } - - public void setQualifiers(String qualifiers) { - this.qualifiers = qualifiers; - } - - @Override - public Boolean isUserConfigurable() { - return userConfigurable; - } - - public void setUserConfigurable(Boolean userConfigurable) { - this.userConfigurable = userConfigurable; - } - - public String getLabel() { - return label; - } - - public void setLabel(String label) { - this.label = label; - } - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description; - } - - public Boolean isPassword() { - return password; - } - - public void setPassword(Boolean password) { - this.password = password; - } - - @Override - public String toString() { - return String.format("PROP - templateId=%s> key=%s value=%s type=%s qual=%s conf=%s label=%s desc=%s password=%s", - templateId, key, value, type, qualifiers, userConfigurable, label, description, password); - } -} diff --git a/engine/schema/src/main/java/com/cloud/storage/VMTemplateDetailVO.java b/engine/schema/src/main/java/com/cloud/storage/VMTemplateDetailVO.java index 5010edfa762..574f4fc500e 100755 --- a/engine/schema/src/main/java/com/cloud/storage/VMTemplateDetailVO.java +++ b/engine/schema/src/main/java/com/cloud/storage/VMTemplateDetailVO.java @@ -21,6 +21,7 @@ import javax.persistence.Entity; import javax.persistence.GeneratedValue; import javax.persistence.GenerationType; import javax.persistence.Id; +import javax.persistence.Lob; import javax.persistence.Table; import org.apache.cloudstack.api.ResourceDetail; @@ -39,7 +40,8 @@ public class VMTemplateDetailVO implements ResourceDetail { @Column(name = "name") private String name; - @Column(name = "value", length = 1024) + @Lob + @Column(name = "value", length = 65535) private String value; @Column(name = "display") diff --git a/engine/schema/src/main/java/com/cloud/storage/VMTemplateStoragePoolVO.java b/engine/schema/src/main/java/com/cloud/storage/VMTemplateStoragePoolVO.java index 6dfe6eb3e87..69c9c85ab5a 100644 --- a/engine/schema/src/main/java/com/cloud/storage/VMTemplateStoragePoolVO.java +++ b/engine/schema/src/main/java/com/cloud/storage/VMTemplateStoragePoolVO.java @@ -95,6 +95,9 @@ public class VMTemplateStoragePoolVO implements VMTemplateStorageResourceAssoc, @Enumerated(EnumType.STRING) ObjectInDataStoreStateMachine.State state; + @Column(name = "deployment_option") + private String deploymentOption; + @Override public String getInstallPath() { return installPath; @@ -168,17 +171,18 @@ public class VMTemplateStoragePoolVO implements VMTemplateStorageResourceAssoc, return downloadState; } - public VMTemplateStoragePoolVO(long poolId, long templateId) { + public VMTemplateStoragePoolVO(long poolId, long templateId, String configuration) { super(); this.poolId = poolId; this.templateId = templateId; this.downloadState = Status.NOT_DOWNLOADED; this.state = ObjectInDataStoreStateMachine.State.Allocated; this.markedForGC = false; + this.deploymentOption = configuration; } public VMTemplateStoragePoolVO(long poolId, long templateId, Date lastUpdated, int downloadPercent, Status downloadState, String localDownloadPath, - String errorString, String jobId, String installPath, long templateSize) { + String errorString, String jobId, String installPath, long templateSize, String configuration) { super(); this.poolId = poolId; this.templateId = templateId; @@ -190,6 +194,7 @@ public class VMTemplateStoragePoolVO implements VMTemplateStorageResourceAssoc, this.jobId = jobId; this.installPath = installPath; this.templateSize = templateSize; + this.deploymentOption = configuration; } protected VMTemplateStoragePoolVO() { @@ -300,4 +305,11 @@ public class VMTemplateStoragePoolVO implements VMTemplateStorageResourceAssoc, return this.state; } + public String getDeploymentOption() { + return deploymentOption; + } + + public void setDeploymentOption(String deploymentOption) { + this.deploymentOption = deploymentOption; + } } diff --git a/engine/schema/src/main/java/com/cloud/storage/VMTemplateVO.java b/engine/schema/src/main/java/com/cloud/storage/VMTemplateVO.java index af04099f9a2..61df40e50d8 100644 --- a/engine/schema/src/main/java/com/cloud/storage/VMTemplateVO.java +++ b/engine/schema/src/main/java/com/cloud/storage/VMTemplateVO.java @@ -152,6 +152,9 @@ public class VMTemplateVO implements VirtualMachineTemplate { @Column(name = "parent_template_id") private Long parentTemplateId; + @Column(name = "deploy_as_is") + private boolean deployAsIs; + @Override public String getUniqueName() { return uniqueName; @@ -192,9 +195,9 @@ public class VMTemplateVO implements VirtualMachineTemplate { uuid = UUID.randomUUID().toString(); } - public VMTemplateVO(long id, String name, ImageFormat format, boolean isPublic, boolean featured, boolean isExtractable, TemplateType type, String url, - boolean requiresHvm, int bits, long accountId, String cksum, String displayText, boolean enablePassword, long guestOSId, boolean bootable, - HypervisorType hyperType, String templateTag, Map details, boolean sshKeyEnabled, boolean isDynamicallyScalable, boolean directDownload) { + public VMTemplateVO(long id, String name, ImageFormat format, boolean isPublic, boolean featured, boolean isExtractable, TemplateType type, String url, boolean requiresHvm, int bits, long accountId, String cksum, String displayText, boolean enablePassword, long guestOSId, boolean bootable, + HypervisorType hyperType, String templateTag, Map details, boolean sshKeyEnabled, boolean isDynamicallyScalable, boolean directDownload, + boolean deployAsIs) { this(id, name, format, @@ -219,6 +222,7 @@ public class VMTemplateVO implements VirtualMachineTemplate { dynamicallyScalable = isDynamicallyScalable; state = State.Active; this.directDownload = directDownload; + this.deployAsIs = deployAsIs; } public static VMTemplateVO createPreHostIso(Long id, String uniqueName, String name, ImageFormat format, boolean isPublic, boolean featured, TemplateType type, @@ -637,4 +641,11 @@ public class VMTemplateVO implements VirtualMachineTemplate { this.parentTemplateId = parentTemplateId; } + @Override public boolean isDeployAsIs() { + return deployAsIs; + } + + public void setDeployAsIs(boolean deployAsIs) { + this.deployAsIs = deployAsIs; + } } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/TemplateOVFPropertiesDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/TemplateOVFPropertiesDaoImpl.java deleted file mode 100644 index cf6a280b034..00000000000 --- a/engine/schema/src/main/java/com/cloud/storage/dao/TemplateOVFPropertiesDaoImpl.java +++ /dev/null @@ -1,78 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package com.cloud.storage.dao; - -import com.cloud.storage.TemplateOVFPropertyVO; -import com.cloud.utils.db.GenericDaoBase; -import com.cloud.utils.db.SearchBuilder; -import com.cloud.utils.db.SearchCriteria; -import com.cloud.utils.db.TransactionLegacy; -import org.apache.commons.collections.CollectionUtils; -import org.apache.log4j.Logger; -import org.springframework.stereotype.Component; - -import java.util.List; - -@Component -public class TemplateOVFPropertiesDaoImpl extends GenericDaoBase implements TemplateOVFPropertiesDao { - - private final static Logger s_logger = Logger.getLogger(TemplateOVFPropertiesDaoImpl.class); - - SearchBuilder OptionsSearchBuilder; - - public TemplateOVFPropertiesDaoImpl() { - super(); - OptionsSearchBuilder = createSearchBuilder(); - OptionsSearchBuilder.and("templateid", OptionsSearchBuilder.entity().getTemplateId(), SearchCriteria.Op.EQ); - OptionsSearchBuilder.and("key", OptionsSearchBuilder.entity().getKey(), SearchCriteria.Op.EQ); - OptionsSearchBuilder.done(); - } - - @Override - public boolean existsOption(long templateId, String key) { - return findByTemplateAndKey(templateId, key) != null; - } - - @Override - public TemplateOVFPropertyVO findByTemplateAndKey(long templateId, String key) { - SearchCriteria sc = OptionsSearchBuilder.create(); - sc.setParameters("templateid", templateId); - sc.setParameters("key", key); - return findOneBy(sc); - } - - @Override - public void saveOptions(List opts) { - if (CollectionUtils.isEmpty(opts)) { - return; - } - TransactionLegacy txn = TransactionLegacy.currentTxn(); - txn.start(); - for (TemplateOVFPropertyVO opt : opts) { - persist(opt); - } - txn.commit(); - } - - @Override - public List listByTemplateId(long templateId) { - SearchCriteria sc = OptionsSearchBuilder.create(); - sc.setParameters("templateid", templateId); - return listBy(sc); - } -} diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDetailsDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDetailsDao.java index fe69630ae2e..51c9cbbbb40 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDetailsDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDetailsDao.java @@ -16,11 +16,25 @@ // under the License. package com.cloud.storage.dao; +import com.cloud.agent.api.storage.OVFPropertyTO; +import com.cloud.agent.api.to.DatadiskTO; +import org.apache.cloudstack.api.net.NetworkPrerequisiteTO; import org.apache.cloudstack.resourcedetail.ResourceDetailsDao; import com.cloud.storage.VMTemplateDetailVO; import com.cloud.utils.db.GenericDao; +import java.util.List; + public interface VMTemplateDetailsDao extends GenericDao, ResourceDetailsDao { + boolean existsOption(long templateId, String key); + OVFPropertyTO findPropertyByTemplateAndKey(long templateId, String key); + void saveOptions(List opts); + List listPropertiesByTemplateId(long templateId); + List listNetworkRequirementsByTemplateId(long templateId); + List listDisksByTemplateId(long templateId); + + List listDetailsByTemplateIdMatchingPrefix(long templateId, String prefix); + String getTemplateEulaSectionsUrl(long templateId); } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDetailsDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDetailsDaoImpl.java index 3e7072f6bf0..d904e1a6de7 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDetailsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDetailsDaoImpl.java @@ -17,17 +17,137 @@ package com.cloud.storage.dao; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; + +import org.apache.cloudstack.api.net.NetworkPrerequisiteTO; +import org.apache.cloudstack.resourcedetail.ResourceDetailsDaoBase; +import org.apache.commons.collections.CollectionUtils; +import org.apache.log4j.Logger; import org.springframework.stereotype.Component; -import org.apache.cloudstack.resourcedetail.ResourceDetailsDaoBase; - +import com.cloud.agent.api.storage.OVFPropertyTO; +import com.cloud.agent.api.to.DatadiskTO; +import com.cloud.storage.ImageStore; import com.cloud.storage.VMTemplateDetailVO; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.db.TransactionLegacy; +import com.google.gson.Gson; @Component public class VMTemplateDetailsDaoImpl extends ResourceDetailsDaoBase implements VMTemplateDetailsDao { + private final static Logger LOGGER = Logger.getLogger(VMTemplateDetailsDaoImpl.class); + + Gson gson = new Gson(); + + SearchBuilder OptionsSearchBuilder; + + public VMTemplateDetailsDaoImpl() { + super(); + OptionsSearchBuilder = createSearchBuilder(); + OptionsSearchBuilder.and("resourceId", OptionsSearchBuilder.entity().getResourceId(), SearchCriteria.Op.EQ); + OptionsSearchBuilder.and("name", OptionsSearchBuilder.entity().getName(), SearchCriteria.Op.EQ); + OptionsSearchBuilder.done(); + } + @Override public void addDetail(long resourceId, String key, String value, boolean display) { super.addDetail(new VMTemplateDetailVO(resourceId, key, value, display)); } -} + + @Override + public boolean existsOption(long templateId, String key) { + return findPropertyByTemplateAndKey(templateId, key) != null; + } + + @Override + public OVFPropertyTO findPropertyByTemplateAndKey(long templateId, String key) { + SearchCriteria sc = OptionsSearchBuilder.create(); + sc.setParameters("resourceId", templateId); + sc.setParameters("name", key.startsWith(ImageStore.ACS_PROPERTY_PREFIX) ? key : ImageStore.ACS_PROPERTY_PREFIX + key); + OVFPropertyTO property = null; + VMTemplateDetailVO detail = findOneBy(sc); + if (detail != null) { + property = gson.fromJson(detail.getValue(), OVFPropertyTO.class); + } + return property; + } + + @Override + public void saveOptions(List opts) { + if (CollectionUtils.isEmpty(opts)) { + return; + } + TransactionLegacy txn = TransactionLegacy.currentTxn(); + txn.start(); + for (OVFPropertyTO opt : opts) { + String json = gson.toJson(opt); + VMTemplateDetailVO templateDetailVO = new VMTemplateDetailVO(opt.getTemplateId(), ImageStore.ACS_PROPERTY_PREFIX + opt.getKey(), json, opt.isUserConfigurable()); + persist(templateDetailVO); + } + txn.commit(); + } + + @Override + public List listPropertiesByTemplateId(long templateId) { + List ovfProperties = listDetailsByTemplateIdMatchingPrefix(templateId, ImageStore.ACS_PROPERTY_PREFIX); + List properties = new ArrayList<>(); + for (VMTemplateDetailVO property : ovfProperties) { + OVFPropertyTO ovfPropertyTO = gson.fromJson(property.getValue(), OVFPropertyTO.class); + properties.add(ovfPropertyTO); + } + return properties; + } + + @Override + public List listNetworkRequirementsByTemplateId(long templateId) { + List networkDetails = listDetailsByTemplateIdMatchingPrefix(templateId, ImageStore.REQUIRED_NETWORK_PREFIX); + List networkPrereqs = new ArrayList<>(); + for (VMTemplateDetailVO property : networkDetails) { + NetworkPrerequisiteTO ovfPropertyTO = gson.fromJson(property.getValue(), NetworkPrerequisiteTO.class); + networkPrereqs.add(ovfPropertyTO); + } + networkPrereqs.sort(new Comparator() { + @Override + public int compare(NetworkPrerequisiteTO o1, NetworkPrerequisiteTO o2) { + return o1.getInstanceID() - o2.getInstanceID(); + } + }); + return networkPrereqs; + } + + @Override + public List listDisksByTemplateId(long templateId) { + List diskDefinitions = listDetailsByTemplateIdMatchingPrefix(templateId, ImageStore.DISK_DEFINITION_PREFIX); + List disks = new ArrayList<>(); + for (VMTemplateDetailVO detail : diskDefinitions) { + DatadiskTO datadiskTO = gson.fromJson(detail.getValue(), DatadiskTO.class); + disks.add(datadiskTO); + } + return disks; + } + + @Override + public List listDetailsByTemplateIdMatchingPrefix(long templateId, String prefix) { + SearchCriteria ssc = createSearchCriteria(); + ssc.addAnd("resourceId", SearchCriteria.Op.EQ, templateId); + ssc.addAnd("name", SearchCriteria.Op.LIKE, prefix + "%"); + + return search(ssc, null); + } + + @Override + public String getTemplateEulaSectionsUrl(long templateId) { + List details = findDetails(templateId, ImageStore.OVF_EULA_SECTION_PREFIX); + if (CollectionUtils.isEmpty(details)) { + return null; + } + if (details.size() > 1) { + LOGGER.error("Multiple details for EULA sections for template " + templateId + " returning one"); + } + return details.get(0).getValue(); + } +} \ No newline at end of file diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplatePoolDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplatePoolDao.java index 05afad67655..d00eeceec7c 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplatePoolDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplatePoolDao.java @@ -27,27 +27,27 @@ import com.cloud.utils.fsm.StateDao; public interface VMTemplatePoolDao extends GenericDao, StateDao { - public List listByPoolId(long id); + List listByPoolId(long id); - public List listByTemplateId(long templateId); + List listByTemplateId(long templateId); - public VMTemplateStoragePoolVO findByPoolTemplate(long poolId, long templateId); + VMTemplateStoragePoolVO findByPoolTemplate(long poolId, long templateId, String configuration); - public List listByPoolIdAndState(long poolId, ObjectInDataStoreStateMachine.State state); + List listByPoolIdAndState(long poolId, ObjectInDataStoreStateMachine.State state); - public List listByTemplateStatus(long templateId, VMTemplateStoragePoolVO.Status downloadState); + List listByTemplateStatus(long templateId, VMTemplateStoragePoolVO.Status downloadState); - public List listByTemplateStatus(long templateId, VMTemplateStoragePoolVO.Status downloadState, long poolId); + List listByTemplateStatus(long templateId, VMTemplateStoragePoolVO.Status downloadState, long poolId); - public List listByTemplateStatus(long templateId, long datacenterId, VMTemplateStoragePoolVO.Status downloadState); + List listByTemplateStatus(long templateId, long datacenterId, VMTemplateStoragePoolVO.Status downloadState); - public List listByTemplateStatus(long templateId, long datacenterId, long podId, VMTemplateStoragePoolVO.Status downloadState); + List listByTemplateStatus(long templateId, long datacenterId, long podId, VMTemplateStoragePoolVO.Status downloadState); - public List listByTemplateStates(long templateId, VMTemplateStoragePoolVO.Status... states); + List listByTemplateStates(long templateId, VMTemplateStoragePoolVO.Status... states); boolean templateAvailable(long templateId, long poolId); - public VMTemplateStoragePoolVO findByHostTemplate(Long hostId, Long templateId); + VMTemplateStoragePoolVO findByHostTemplate(Long hostId, Long templateId, String configuration); VMTemplateStoragePoolVO findByPoolPath(Long poolId, String path); diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplatePoolDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplatePoolDaoImpl.java index 32874701128..998df5e7987 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplatePoolDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplatePoolDaoImpl.java @@ -28,6 +28,7 @@ import javax.inject.Inject; import org.apache.cloudstack.engine.subsystem.api.storage.DataObjectInStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; +import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -73,7 +74,7 @@ public class VMTemplatePoolDaoImpl extends GenericDaoBase sc = PoolTemplateSearch.create(); sc.setParameters("pool_id", poolId); sc.setParameters("template_id", templateId); + if (StringUtils.isNotBlank(configuration)) { + sc.setParameters("configuration", configuration); + } return findOneIncludingRemovedBy(sc); } @@ -219,13 +224,17 @@ public class VMTemplatePoolDaoImpl extends GenericDaoBase listByHostTemplate(long hostId, long templateId) { + public List listByHostTemplate(long hostId, long templateId, String configuration) { TransactionLegacy txn = TransactionLegacy.currentTxn(); List result = new ArrayList(); String sql = HOST_TEMPLATE_SEARCH; + sql += StringUtils.isBlank(configuration) ? "IS NULL" : "= ?"; try(PreparedStatement pstmt = txn.prepareStatement(sql);) { pstmt.setLong(1, hostId); pstmt.setLong(2, templateId); + if (StringUtils.isNotBlank(configuration)) { + pstmt.setString(3, configuration); + } try(ResultSet rs = pstmt.executeQuery();) { while (rs.next()) { // result.add(toEntityBean(rs, false)); TODO: this is buggy in @@ -245,7 +254,7 @@ public class VMTemplatePoolDaoImpl extends GenericDaoBase result = listByHostTemplate(hostId, templateId); - return (result.size() == 0) ? null : result.get(1); + public VMTemplateStoragePoolVO findByHostTemplate(Long hostId, Long templateId, String configuration) { + List result = listByHostTemplate(hostId, templateId, configuration); + return (result.size() == 0) ? null : result.get(0); } @Override diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/UserVmDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/UserVmDaoImpl.java index 25479d6658a..6bf0dc23168 100644 --- a/engine/schema/src/main/java/com/cloud/vm/dao/UserVmDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/UserVmDaoImpl.java @@ -30,6 +30,7 @@ import javax.annotation.PostConstruct; import javax.inject.Inject; import com.cloud.offerings.dao.NetworkOfferingServiceMapDao; +import com.cloud.storage.ImageStore; import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; @@ -382,13 +383,25 @@ public class UserVmDaoImpl extends GenericDaoBase implements Use List details = new ArrayList(); for (Map.Entry entry : detailsStr.entrySet()) { - boolean display = visibilityMap.getOrDefault(entry.getKey(), true); + boolean display = visibilityMap.getOrDefault(entry.getKey(), true) && displayOVFDetails(entry.getKey()); details.add(new UserVmDetailVO(vm.getId(), entry.getKey(), entry.getValue(), display)); } _detailsDao.saveDetails(details); } + /* + Do not display VM properties parsed from OVF, handled internally + */ + private boolean displayOVFDetails(String key) { + if (key.startsWith(ImageStore.ACS_PROPERTY_PREFIX) || key.startsWith(ImageStore.OVF_HARDWARE_ITEM_PREFIX) || + key.startsWith(ImageStore.OVF_HARDWARE_CONFIGURATION_PREFIX) || key.startsWith(ImageStore.DISK_DEFINITION_PREFIX) || + key.startsWith(ImageStore.REQUIRED_NETWORK_PREFIX) || key.startsWith(ImageStore.OVF_EULA_SECTION_PREFIX)) { + return false; + } + return true; + } + @Override public List listPodIdsHavingVmsforAccount(long zoneId, long accountId) { TransactionLegacy txn = TransactionLegacy.currentTxn(); diff --git a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml index 052a3dfc205..cde3518e7d2 100644 --- a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml +++ b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml @@ -293,7 +293,6 @@ - diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql b/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql index 4c7c7c994da..b7e94cc0e86 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql @@ -260,6 +260,218 @@ CREATE VIEW `cloud`.`storage_pool_view` AS LEFT JOIN `async_job` ON (((`async_job`.`instance_id` = `storage_pool`.`id`) AND (`async_job`.`instance_type` = 'StoragePool') AND (`async_job`.`job_status` = 0)))); +-- Add passthrough instruction for appliance deployments +ALTER TABLE `cloud`.`vm_template` ADD COLUMN `deploy_as_is` tinyint(1) NOT NULL DEFAULT '0' COMMENT 'True if the template should be deployed with disks and networks as defined by OVF'; + +-- Extend the template details value field +ALTER TABLE `cloud`.`vm_template_details` MODIFY COLUMN `value` TEXT CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL; + +-- Changes to template_view for both deploying multidisk OVA/vApp as is +DROP VIEW IF EXISTS `cloud`.`template_view`; +CREATE VIEW `cloud`.`template_view` AS + SELECT + `vm_template`.`id` AS `id`, + `vm_template`.`uuid` AS `uuid`, + `vm_template`.`unique_name` AS `unique_name`, + `vm_template`.`name` AS `name`, + `vm_template`.`public` AS `public`, + `vm_template`.`featured` AS `featured`, + `vm_template`.`type` AS `type`, + `vm_template`.`hvm` AS `hvm`, + `vm_template`.`bits` AS `bits`, + `vm_template`.`url` AS `url`, + `vm_template`.`format` AS `format`, + `vm_template`.`created` AS `created`, + `vm_template`.`checksum` AS `checksum`, + `vm_template`.`display_text` AS `display_text`, + `vm_template`.`enable_password` AS `enable_password`, + `vm_template`.`dynamically_scalable` AS `dynamically_scalable`, + `vm_template`.`state` AS `template_state`, + `vm_template`.`guest_os_id` AS `guest_os_id`, + `guest_os`.`uuid` AS `guest_os_uuid`, + `guest_os`.`display_name` AS `guest_os_name`, + `vm_template`.`bootable` AS `bootable`, + `vm_template`.`prepopulate` AS `prepopulate`, + `vm_template`.`cross_zones` AS `cross_zones`, + `vm_template`.`hypervisor_type` AS `hypervisor_type`, + `vm_template`.`extractable` AS `extractable`, + `vm_template`.`template_tag` AS `template_tag`, + `vm_template`.`sort_key` AS `sort_key`, + `vm_template`.`removed` AS `removed`, + `vm_template`.`enable_sshkey` AS `enable_sshkey`, + `parent_template`.`id` AS `parent_template_id`, + `parent_template`.`uuid` AS `parent_template_uuid`, + `source_template`.`id` AS `source_template_id`, + `source_template`.`uuid` AS `source_template_uuid`, + `account`.`id` AS `account_id`, + `account`.`uuid` AS `account_uuid`, + `account`.`account_name` AS `account_name`, + `account`.`type` AS `account_type`, + `domain`.`id` AS `domain_id`, + `domain`.`uuid` AS `domain_uuid`, + `domain`.`name` AS `domain_name`, + `domain`.`path` AS `domain_path`, + `projects`.`id` AS `project_id`, + `projects`.`uuid` AS `project_uuid`, + `projects`.`name` AS `project_name`, + `data_center`.`id` AS `data_center_id`, + `data_center`.`uuid` AS `data_center_uuid`, + `data_center`.`name` AS `data_center_name`, + `launch_permission`.`account_id` AS `lp_account_id`, + `template_store_ref`.`store_id` AS `store_id`, + `image_store`.`scope` AS `store_scope`, + `template_store_ref`.`state` AS `state`, + `template_store_ref`.`download_state` AS `download_state`, + `template_store_ref`.`download_pct` AS `download_pct`, + `template_store_ref`.`error_str` AS `error_str`, + `template_store_ref`.`size` AS `size`, + `template_store_ref`.physical_size AS `physical_size`, + `template_store_ref`.`destroyed` AS `destroyed`, + `template_store_ref`.`created` AS `created_on_store`, + `vm_template_details`.`name` AS `detail_name`, + `vm_template_details`.`value` AS `detail_value`, + `resource_tags`.`id` AS `tag_id`, + `resource_tags`.`uuid` AS `tag_uuid`, + `resource_tags`.`key` AS `tag_key`, + `resource_tags`.`value` AS `tag_value`, + `resource_tags`.`domain_id` AS `tag_domain_id`, + `domain`.`uuid` AS `tag_domain_uuid`, + `domain`.`name` AS `tag_domain_name`, + `resource_tags`.`account_id` AS `tag_account_id`, + `account`.`account_name` AS `tag_account_name`, + `resource_tags`.`resource_id` AS `tag_resource_id`, + `resource_tags`.`resource_uuid` AS `tag_resource_uuid`, + `resource_tags`.`resource_type` AS `tag_resource_type`, + `resource_tags`.`customer` AS `tag_customer`, + CONCAT(`vm_template`.`id`, + '_', + IFNULL(`data_center`.`id`, 0)) AS `temp_zone_pair`, + `vm_template`.`direct_download` AS `direct_download`, + `vm_template`.`deploy_as_is` AS `deploy_as_is` + FROM + (((((((((((((`vm_template` + JOIN `guest_os` ON ((`guest_os`.`id` = `vm_template`.`guest_os_id`))) + JOIN `account` ON ((`account`.`id` = `vm_template`.`account_id`))) + JOIN `domain` ON ((`domain`.`id` = `account`.`domain_id`))) + LEFT JOIN `projects` ON ((`projects`.`project_account_id` = `account`.`id`))) + LEFT JOIN `vm_template_details` ON ((`vm_template_details`.`template_id` = `vm_template`.`id`))) + LEFT JOIN `vm_template` `source_template` ON ((`source_template`.`id` = `vm_template`.`source_template_id`))) + LEFT JOIN `template_store_ref` ON (((`template_store_ref`.`template_id` = `vm_template`.`id`) + AND (`template_store_ref`.`store_role` = 'Image') + AND (`template_store_ref`.`destroyed` = 0)))) + LEFT JOIN `vm_template` `parent_template` ON ((`parent_template`.`id` = `vm_template`.`parent_template_id`))) + LEFT JOIN `image_store` ON ((ISNULL(`image_store`.`removed`) + AND (`template_store_ref`.`store_id` IS NOT NULL) + AND (`image_store`.`id` = `template_store_ref`.`store_id`)))) + LEFT JOIN `template_zone_ref` ON (((`template_zone_ref`.`template_id` = `vm_template`.`id`) + AND ISNULL(`template_store_ref`.`store_id`) + AND ISNULL(`template_zone_ref`.`removed`)))) + LEFT JOIN `data_center` ON (((`image_store`.`data_center_id` = `data_center`.`id`) + OR (`template_zone_ref`.`zone_id` = `data_center`.`id`)))) + LEFT JOIN `launch_permission` ON ((`launch_permission`.`template_id` = `vm_template`.`id`))) + LEFT JOIN `resource_tags` ON (((`resource_tags`.`resource_id` = `vm_template`.`id`) + AND ((`resource_tags`.`resource_type` = 'Template') + OR (`resource_tags`.`resource_type` = 'ISO'))))); + +-- Add mincpu, maxcpu, minmemory and maxmemory to the view supporting constrained offerings +DROP VIEW IF EXISTS `cloud`.`service_offering_view`; +CREATE VIEW `cloud`.`service_offering_view` AS + SELECT + `service_offering`.`id` AS `id`, + `disk_offering`.`uuid` AS `uuid`, + `disk_offering`.`name` AS `name`, + `disk_offering`.`display_text` AS `display_text`, + `disk_offering`.`provisioning_type` AS `provisioning_type`, + `disk_offering`.`created` AS `created`, + `disk_offering`.`tags` AS `tags`, + `disk_offering`.`removed` AS `removed`, + `disk_offering`.`use_local_storage` AS `use_local_storage`, + `disk_offering`.`system_use` AS `system_use`, + `disk_offering`.`customized_iops` AS `customized_iops`, + `disk_offering`.`min_iops` AS `min_iops`, + `disk_offering`.`max_iops` AS `max_iops`, + `disk_offering`.`hv_ss_reserve` AS `hv_ss_reserve`, + `disk_offering`.`bytes_read_rate` AS `bytes_read_rate`, + `disk_offering`.`bytes_read_rate_max` AS `bytes_read_rate_max`, + `disk_offering`.`bytes_read_rate_max_length` AS `bytes_read_rate_max_length`, + `disk_offering`.`bytes_write_rate` AS `bytes_write_rate`, + `disk_offering`.`bytes_write_rate_max` AS `bytes_write_rate_max`, + `disk_offering`.`bytes_write_rate_max_length` AS `bytes_write_rate_max_length`, + `disk_offering`.`iops_read_rate` AS `iops_read_rate`, + `disk_offering`.`iops_read_rate_max` AS `iops_read_rate_max`, + `disk_offering`.`iops_read_rate_max_length` AS `iops_read_rate_max_length`, + `disk_offering`.`iops_write_rate` AS `iops_write_rate`, + `disk_offering`.`iops_write_rate_max` AS `iops_write_rate_max`, + `disk_offering`.`iops_write_rate_max_length` AS `iops_write_rate_max_length`, + `disk_offering`.`cache_mode` AS `cache_mode`, + `service_offering`.`cpu` AS `cpu`, + `service_offering`.`speed` AS `speed`, + `service_offering`.`ram_size` AS `ram_size`, + `service_offering`.`nw_rate` AS `nw_rate`, + `service_offering`.`mc_rate` AS `mc_rate`, + `service_offering`.`ha_enabled` AS `ha_enabled`, + `service_offering`.`limit_cpu_use` AS `limit_cpu_use`, + `service_offering`.`host_tag` AS `host_tag`, + `service_offering`.`default_use` AS `default_use`, + `service_offering`.`vm_type` AS `vm_type`, + `service_offering`.`sort_key` AS `sort_key`, + `service_offering`.`is_volatile` AS `is_volatile`, + `service_offering`.`deployment_planner` AS `deployment_planner`, + GROUP_CONCAT(DISTINCT(domain.id)) AS domain_id, + GROUP_CONCAT(DISTINCT(domain.uuid)) AS domain_uuid, + GROUP_CONCAT(DISTINCT(domain.name)) AS domain_name, + GROUP_CONCAT(DISTINCT(domain.path)) AS domain_path, + GROUP_CONCAT(DISTINCT(zone.id)) AS zone_id, + GROUP_CONCAT(DISTINCT(zone.uuid)) AS zone_uuid, + GROUP_CONCAT(DISTINCT(zone.name)) AS zone_name, + IFNULL(`min_compute_details`.`value`, `cpu`) AS min_cpu, + IFNULL(`max_compute_details`.`value`, `cpu`) AS max_cpu, + IFNULL(`min_memory_details`.`value`, `ram_size`) AS min_memory, + IFNULL(`max_memory_details`.`value`, `ram_size`) AS max_memory + FROM + `cloud`.`service_offering` + INNER JOIN + `cloud`.`disk_offering_view` AS `disk_offering` ON service_offering.id = disk_offering.id + LEFT JOIN + `cloud`.`service_offering_details` AS `domain_details` ON `domain_details`.`service_offering_id` = `disk_offering`.`id` AND `domain_details`.`name`='domainid' + LEFT JOIN + `cloud`.`domain` AS `domain` ON FIND_IN_SET(`domain`.`id`, `domain_details`.`value`) + LEFT JOIN + `cloud`.`service_offering_details` AS `zone_details` ON `zone_details`.`service_offering_id` = `disk_offering`.`id` AND `zone_details`.`name`='zoneid' + LEFT JOIN + `cloud`.`data_center` AS `zone` ON FIND_IN_SET(`zone`.`id`, `zone_details`.`value`) + LEFT JOIN + `cloud`.`service_offering_details` AS `min_compute_details` ON `min_compute_details`.`service_offering_id` = `disk_offering`.`id` + AND `min_compute_details`.`name` = 'mincpunumber' + LEFT JOIN + `cloud`.`service_offering_details` AS `max_compute_details` ON `max_compute_details`.`service_offering_id` = `disk_offering`.`id` + AND `max_compute_details`.`name` = 'maxcpunumber' + LEFT JOIN + `cloud`.`service_offering_details` AS `min_memory_details` ON `min_memory_details`.`service_offering_id` = `disk_offering`.`id` + AND `min_memory_details`.`name` = 'minmemory' + LEFT JOIN + `cloud`.`service_offering_details` AS `max_memory_details` ON `max_memory_details`.`service_offering_id` = `disk_offering`.`id` + AND `max_memory_details`.`name` = 'maxmemory' + WHERE + `disk_offering`.`state`='Active' + GROUP BY + `service_offering`.`id`; + +ALTER TABLE `cloud`.`template_spool_ref` +DROP FOREIGN KEY `fk_template_spool_ref__template_id`; + +ALTER TABLE `cloud`.`template_spool_ref` +ADD COLUMN `deployment_option` VARCHAR(255) NULL DEFAULT NULL AFTER `updated`, +ADD INDEX `fk_template_spool_ref__template_id_idx` (`template_id` ASC), +ADD UNIQUE INDEX `index_template_spool_configuration` (`pool_id` ASC, `template_id` ASC, `deployment_option` ASC), +DROP INDEX `i_template_spool_ref__template_id__pool_id` ; + +ALTER TABLE `cloud`.`template_spool_ref` +ADD CONSTRAINT `fk_template_spool_ref__template_id` + FOREIGN KEY (`template_id`) + REFERENCES `cloud`.`vm_template` (`id`) + ON DELETE NO ACTION + ON UPDATE NO ACTION; ALTER TABLE `cloud`.`image_store` ADD COLUMN `readonly` boolean DEFAULT false COMMENT 'defines status of image store'; @@ -285,4 +497,4 @@ ALTER VIEW `cloud`.`image_store_view` AS left join `cloud`.`data_center` ON image_store.data_center_id = data_center.id left join - `cloud`.`image_store_details` ON image_store_details.store_id = image_store.id; + `cloud`.`image_store_details` ON image_store_details.store_id = image_store.id; \ No newline at end of file diff --git a/engine/storage/cache/src/main/java/org/apache/cloudstack/storage/cache/manager/StorageCacheManagerImpl.java b/engine/storage/cache/src/main/java/org/apache/cloudstack/storage/cache/manager/StorageCacheManagerImpl.java index 278c80d3247..40eeafef692 100644 --- a/engine/storage/cache/src/main/java/org/apache/cloudstack/storage/cache/manager/StorageCacheManagerImpl.java +++ b/engine/storage/cache/src/main/java/org/apache/cloudstack/storage/cache/manager/StorageCacheManagerImpl.java @@ -291,7 +291,7 @@ public class StorageCacheManagerImpl implements StorageCacheManager, Manager { if (st == ObjectInDataStoreStateMachine.State.Ready) { s_logger.debug("there is already one in the cache store"); - DataObject dataObj = objectInStoreMgr.get(data, store); + DataObject dataObj = objectInStoreMgr.get(data, store, null); dataObj.incRefCount(); existingDataObj = dataObj; } diff --git a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageDataMotionStrategy.java b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageDataMotionStrategy.java index e6b5c85b924..971859685ff 100644 --- a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageDataMotionStrategy.java +++ b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageDataMotionStrategy.java @@ -195,7 +195,7 @@ public class KvmNonManagedStorageDataMotionStrategy extends StorageSystemDataMot @Override protected void copyTemplateToTargetFilesystemStorageIfNeeded(VolumeInfo srcVolumeInfo, StoragePool srcStoragePool, DataStore destDataStore, StoragePool destStoragePool, Host destHost) { - VMTemplateStoragePoolVO sourceVolumeTemplateStoragePoolVO = vmTemplatePoolDao.findByPoolTemplate(destStoragePool.getId(), srcVolumeInfo.getTemplateId()); + VMTemplateStoragePoolVO sourceVolumeTemplateStoragePoolVO = vmTemplatePoolDao.findByPoolTemplate(destStoragePool.getId(), srcVolumeInfo.getTemplateId(), null); if (sourceVolumeTemplateStoragePoolVO == null && destStoragePool.getPoolType() == StoragePoolType.Filesystem) { DataStore sourceTemplateDataStore = dataStoreManagerImpl.getRandomImageStore(srcVolumeInfo.getDataCenterId()); if (sourceTemplateDataStore != null) { @@ -220,8 +220,8 @@ public class KvmNonManagedStorageDataMotionStrategy extends StorageSystemDataMot * Update the template reference on table "template_spool_ref" (VMTemplateStoragePoolVO). */ protected void updateTemplateReferenceIfSuccessfulCopy(VolumeInfo srcVolumeInfo, StoragePool srcStoragePool, TemplateInfo destTemplateInfo, DataStore destDataStore) { - VMTemplateStoragePoolVO srcVolumeTemplateStoragePoolVO = vmTemplatePoolDao.findByPoolTemplate(srcStoragePool.getId(), srcVolumeInfo.getTemplateId()); - VMTemplateStoragePoolVO destVolumeTemplateStoragePoolVO = new VMTemplateStoragePoolVO(destDataStore.getId(), srcVolumeInfo.getTemplateId()); + VMTemplateStoragePoolVO srcVolumeTemplateStoragePoolVO = vmTemplatePoolDao.findByPoolTemplate(srcStoragePool.getId(), srcVolumeInfo.getTemplateId(), null); + VMTemplateStoragePoolVO destVolumeTemplateStoragePoolVO = new VMTemplateStoragePoolVO(destDataStore.getId(), srcVolumeInfo.getTemplateId(), null); destVolumeTemplateStoragePoolVO.setDownloadPercent(100); destVolumeTemplateStoragePoolVO.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOADED); destVolumeTemplateStoragePoolVO.setState(ObjectInDataStoreStateMachine.State.Ready); diff --git a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java index 4d3ec184ac1..936f0626af3 100644 --- a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java +++ b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java @@ -1710,7 +1710,7 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { * Return expected MigrationOptions for a linked clone volume live storage migration */ protected MigrationOptions createLinkedCloneMigrationOptions(VolumeInfo srcVolumeInfo, VolumeInfo destVolumeInfo, String srcVolumeBackingFile, String srcPoolUuid, Storage.StoragePoolType srcPoolType) { - VMTemplateStoragePoolVO ref = templatePoolDao.findByPoolTemplate(destVolumeInfo.getPoolId(), srcVolumeInfo.getTemplateId()); + VMTemplateStoragePoolVO ref = templatePoolDao.findByPoolTemplate(destVolumeInfo.getPoolId(), srcVolumeInfo.getTemplateId(), null); boolean updateBackingFileReference = ref == null; String backingFile = ref != null ? ref.getInstallPath() : srcVolumeBackingFile; return new MigrationOptions(srcPoolUuid, srcPoolType, backingFile, updateBackingFileReference); @@ -1983,7 +1983,7 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { srcVolumeInfo.getTemplateId() != null && srcVolumeInfo.getPoolId() != null) { VMTemplateVO template = _vmTemplateDao.findById(srcVolumeInfo.getTemplateId()); if (template.getFormat() != null && template.getFormat() != Storage.ImageFormat.ISO) { - VMTemplateStoragePoolVO ref = templatePoolDao.findByPoolTemplate(srcVolumeInfo.getPoolId(), srcVolumeInfo.getTemplateId()); + VMTemplateStoragePoolVO ref = templatePoolDao.findByPoolTemplate(srcVolumeInfo.getPoolId(), srcVolumeInfo.getTemplateId(), null); return ref != null ? ref.getInstallPath() : null; } } @@ -2157,8 +2157,8 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { * Update reference on template_spool_ref table of copied template to destination storage */ protected void updateCopiedTemplateReference(VolumeInfo srcVolumeInfo, VolumeInfo destVolumeInfo) { - VMTemplateStoragePoolVO ref = templatePoolDao.findByPoolTemplate(srcVolumeInfo.getPoolId(), srcVolumeInfo.getTemplateId()); - VMTemplateStoragePoolVO newRef = new VMTemplateStoragePoolVO(destVolumeInfo.getPoolId(), ref.getTemplateId()); + VMTemplateStoragePoolVO ref = templatePoolDao.findByPoolTemplate(srcVolumeInfo.getPoolId(), srcVolumeInfo.getTemplateId(), null); + VMTemplateStoragePoolVO newRef = new VMTemplateStoragePoolVO(destVolumeInfo.getPoolId(), ref.getTemplateId(), null); newRef.setDownloadPercent(100); newRef.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOADED); newRef.setState(ObjectInDataStoreStateMachine.State.Ready); diff --git a/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageSystemDataMotionTest.java b/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageSystemDataMotionTest.java index 6971444bcad..ba7fb74da1d 100644 --- a/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageSystemDataMotionTest.java +++ b/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageSystemDataMotionTest.java @@ -19,6 +19,7 @@ package org.apache.cloudstack.storage.motion; import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.nullable; import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.when; @@ -301,7 +302,7 @@ public class KvmNonManagedStorageSystemDataMotionTest { public void copyTemplateToTargetStorageIfNeededTestTemplateAlreadyOnTargetHost() throws AgentUnavailableException, OperationTimedoutException { Answer copyCommandAnswer = Mockito.mock(Answer.class); Mockito.lenient().when(copyCommandAnswer.getResult()).thenReturn(true); - configureAndTestcopyTemplateToTargetStorageIfNeeded(new VMTemplateStoragePoolVO(0l, 0l), StoragePoolType.Filesystem, 0); + configureAndTestcopyTemplateToTargetStorageIfNeeded(new VMTemplateStoragePoolVO(0l, 0l, null), StoragePoolType.Filesystem, 0); } @Test @@ -316,7 +317,7 @@ public class KvmNonManagedStorageSystemDataMotionTest { if (storagePoolTypeArray[i] == StoragePoolType.Filesystem) { continue; } - configureAndTestcopyTemplateToTargetStorageIfNeeded(new VMTemplateStoragePoolVO(0l, 0l), storagePoolTypeArray[i], 0); + configureAndTestcopyTemplateToTargetStorageIfNeeded(new VMTemplateStoragePoolVO(0l, 0l, null), storagePoolTypeArray[i], 0); } } @@ -353,7 +354,7 @@ public class KvmNonManagedStorageSystemDataMotionTest { Mockito.when(sourceTemplateInfo.getSize()).thenReturn(0l); Mockito.when(sourceTemplateInfo.getHypervisorType()).thenReturn(HypervisorType.KVM); - Mockito.when(vmTemplatePoolDao.findByPoolTemplate(Mockito.anyLong(), Mockito.anyLong())).thenReturn(vmTemplateStoragePoolVO); + Mockito.when(vmTemplatePoolDao.findByPoolTemplate(Mockito.anyLong(), Mockito.anyLong(), nullable(String.class))).thenReturn(vmTemplateStoragePoolVO); Mockito.when(dataStoreManagerImpl.getRandomImageStore(Mockito.anyLong())).thenReturn(sourceTemplateDataStore); Mockito.when(templateDataFactory.getTemplate(Mockito.anyLong(), Mockito.eq(sourceTemplateDataStore))).thenReturn(sourceTemplateInfo); Mockito.when(templateDataFactory.getTemplate(Mockito.anyLong(), Mockito.eq(destDataStore))).thenReturn(sourceTemplateInfo); @@ -362,7 +363,7 @@ public class KvmNonManagedStorageSystemDataMotionTest { Mockito.any(TemplateInfo.class), Mockito.any(DataStore.class)); InOrder verifyInOrder = Mockito.inOrder(vmTemplatePoolDao, dataStoreManagerImpl, templateDataFactory, kvmNonManagedStorageDataMotionStrategy); - verifyInOrder.verify(vmTemplatePoolDao, Mockito.times(1)).findByPoolTemplate(Mockito.anyLong(), Mockito.anyLong()); + verifyInOrder.verify(vmTemplatePoolDao, Mockito.times(1)).findByPoolTemplate(Mockito.anyLong(), Mockito.anyLong(), nullable(String.class)); verifyInOrder.verify(dataStoreManagerImpl, Mockito.times(times)).getRandomImageStore(Mockito.anyLong()); verifyInOrder.verify(templateDataFactory, Mockito.times(times)).getTemplate(Mockito.anyLong(), Mockito.eq(sourceTemplateDataStore)); verifyInOrder.verify(templateDataFactory, Mockito.times(times)).getTemplate(Mockito.anyLong(), Mockito.eq(destDataStore)); diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java index 043af9a49ac..1590fe0bf7d 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java @@ -23,6 +23,9 @@ import java.util.List; import javax.inject.Inject; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.utils.StringUtils; +import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.direct.download.DirectDownloadManager; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; @@ -39,13 +42,11 @@ import org.springframework.stereotype.Component; import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; -import com.cloud.hypervisor.Hypervisor; import com.cloud.storage.DataStoreRole; import com.cloud.storage.VMTemplateStoragePoolVO; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VMTemplatePoolDao; -import com.cloud.utils.exception.CloudRuntimeException; @Component public class TemplateDataFactoryImpl implements TemplateDataFactory { @@ -65,17 +66,30 @@ public class TemplateDataFactoryImpl implements TemplateDataFactory { @Inject PrimaryDataStoreDao primaryDataStoreDao; + @Override + public TemplateInfo getTemplateOnPrimaryStorage(long templateId, DataStore store, String configuration) { + VMTemplateVO templ = imageDataDao.findById(templateId); + if (store.getRole() == DataStoreRole.Primary) { + VMTemplateStoragePoolVO templatePoolVO = templatePoolDao.findByPoolTemplate(store.getId(), templateId, configuration); + if (templatePoolVO != null) { + String deployAsIsConfiguration = templatePoolVO.getDeploymentOption(); + return TemplateObject.getTemplate(templ, store, deployAsIsConfiguration); + } + } + return null; + } + @Override public TemplateInfo getTemplate(long templateId, DataStore store) { VMTemplateVO templ = imageDataDao.findById(templateId); if (store == null && !templ.isDirectDownload()) { - TemplateObject tmpl = TemplateObject.getTemplate(templ, null); + TemplateObject tmpl = TemplateObject.getTemplate(templ, null, null); return tmpl; } // verify if the given input parameters are consistent with our db data. boolean found = false; if (store.getRole() == DataStoreRole.Primary) { - VMTemplateStoragePoolVO templatePoolVO = templatePoolDao.findByPoolTemplate(store.getId(), templateId); + VMTemplateStoragePoolVO templatePoolVO = templatePoolDao.findByPoolTemplate(store.getId(), templateId, null); if (templatePoolVO != null) { found = true; } @@ -94,7 +108,7 @@ public class TemplateDataFactoryImpl implements TemplateDataFactory { } } - TemplateObject tmpl = TemplateObject.getTemplate(templ, store); + TemplateObject tmpl = TemplateObject.getTemplate(templ, store, null); return tmpl; } @@ -130,8 +144,13 @@ public class TemplateDataFactoryImpl implements TemplateDataFactory { } @Override - public TemplateInfo getTemplate(DataObject obj, DataStore store) { - TemplateObject tmpObj = (TemplateObject)this.getTemplate(obj.getId(), store); + public TemplateInfo getTemplate(DataObject obj, DataStore store, String configuration) { + TemplateObject tmpObj; + if (StringUtils.isNotBlank(configuration)) { + tmpObj = (TemplateObject)this.getTemplateOnPrimaryStorage(obj.getId(), store, configuration); + } else { + tmpObj = (TemplateObject)this.getTemplate(obj.getId(), store); + } // carry over url set in passed in data object, for copyTemplate case // where url is generated on demand and not persisted in DB. // need to think of a more generic way to pass these runtime information @@ -217,7 +236,7 @@ public class TemplateDataFactoryImpl implements TemplateDataFactory { if (pool == null) { throw new CloudRuntimeException("No storage pool found where to download template: " + templateId); } - VMTemplateStoragePoolVO spoolRef = templatePoolDao.findByPoolTemplate(pool, templateId); + VMTemplateStoragePoolVO spoolRef = templatePoolDao.findByPoolTemplate(pool, templateId, null); if (spoolRef == null) { directDownloadManager.downloadTemplate(templateId, pool, hostId); } diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java index 00bc7e4208b..ed9359d952a 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java @@ -204,7 +204,7 @@ public class TemplateServiceImpl implements TemplateService { // clean up already persisted template_store_ref entry in case of createTemplateCallback is never called TemplateDataStoreVO templateStoreVO = _vmTemplateStoreDao.findByStoreTemplate(store.getId(), template.getId()); if (templateStoreVO != null) { - TemplateInfo tmplObj = _templateFactory.getTemplate(template, store); + TemplateInfo tmplObj = _templateFactory.getTemplate(template, store, null); tmplObj.processEvent(ObjectInDataStoreStateMachine.Event.OperationFailed); } TemplateApiResult result = new TemplateApiResult(template); @@ -412,7 +412,7 @@ public class TemplateServiceImpl implements TemplateService { VirtualMachineTemplate.Event event = VirtualMachineTemplate.Event.OperationSucceeded; // For multi-disk OVA, check and create data disk templates if (tmplt.getFormat().equals(ImageFormat.OVA)) { - if (!createOvaDataDiskTemplates(_templateFactory.getTemplate(tmlpt.getId(), store))) { + if (!createOvaDataDiskTemplates(_templateFactory.getTemplate(tmlpt.getId(), store), tmplt.isDeployAsIs())) { event = VirtualMachineTemplate.Event.OperationFailed; } } @@ -710,7 +710,7 @@ public class TemplateServiceImpl implements TemplateService { // For multi-disk OVA, check and create data disk templates if (template.getFormat().equals(ImageFormat.OVA)) { - if (!createOvaDataDiskTemplates(template)) { + if (!createOvaDataDiskTemplates(template, template.isDeployAsIs())) { template.processEvent(ObjectInDataStoreStateMachine.Event.OperationFailed); result.setResult(callbackResult.getResult()); if (parentCallback != null) { @@ -737,12 +737,18 @@ public class TemplateServiceImpl implements TemplateService { } @Override - public boolean createOvaDataDiskTemplates(TemplateInfo parentTemplate) { + public List getTemplateDatadisksOnImageStore(TemplateInfo templateInfo, String configurationId) { + ImageStoreEntity tmpltStore = (ImageStoreEntity)templateInfo.getDataStore(); + return tmpltStore.getDataDiskTemplates(templateInfo, configurationId); + } + + @Override + public boolean createOvaDataDiskTemplates(TemplateInfo parentTemplate, boolean deployAsIs) { try { // Get Datadisk template (if any) for OVA List dataDiskTemplates = new ArrayList(); ImageStoreEntity tmpltStore = (ImageStoreEntity)parentTemplate.getDataStore(); - dataDiskTemplates = tmpltStore.getDataDiskTemplates(parentTemplate); + dataDiskTemplates = tmpltStore.getDataDiskTemplates(parentTemplate, null); int diskCount = 0; VMTemplateVO templateVO = _templateDao.findById(parentTemplate.getId()); _templateDao.loadDetails(templateVO); @@ -754,23 +760,27 @@ public class TemplateServiceImpl implements TemplateService { details = new HashMap<>(); } } + for (DatadiskTO diskTemplate : dataDiskTemplates) { - if (!diskTemplate.isBootable()) { - createChildDataDiskTemplate(diskTemplate, templateVO, parentTemplate, imageStore, diskCount++); - if (!diskTemplate.isIso() && Strings.isNullOrEmpty(details.get(VmDetailConstants.DATA_DISK_CONTROLLER))){ - details.put(VmDetailConstants.DATA_DISK_CONTROLLER, getOvaDiskControllerDetails(diskTemplate, false)); - details.put(VmDetailConstants.DATA_DISK_CONTROLLER + diskTemplate.getDiskId(), getOvaDiskControllerDetails(diskTemplate, false)); - } - } else { - finalizeParentTemplate(diskTemplate, templateVO, parentTemplate, imageStore, diskCount++); - if (Strings.isNullOrEmpty(VmDetailConstants.ROOT_DISK_CONTROLLER)) { - final String rootDiskController = getOvaDiskControllerDetails(diskTemplate, true); - if (!Strings.isNullOrEmpty(rootDiskController)) { - details.put(VmDetailConstants.ROOT_DISK_CONTROLLER, rootDiskController); + if (!deployAsIs) { + if (!diskTemplate.isBootable()) { + createChildDataDiskTemplate(diskTemplate, templateVO, parentTemplate, imageStore, diskCount++); + if (!diskTemplate.isIso() && Strings.isNullOrEmpty(details.get(VmDetailConstants.DATA_DISK_CONTROLLER))){ + details.put(VmDetailConstants.DATA_DISK_CONTROLLER, getOvaDiskControllerDetails(diskTemplate, false)); + details.put(VmDetailConstants.DATA_DISK_CONTROLLER + diskTemplate.getDiskId(), getOvaDiskControllerDetails(diskTemplate, false)); + } + } else { + finalizeParentTemplate(diskTemplate, templateVO, parentTemplate, imageStore, diskCount++); + if (Strings.isNullOrEmpty(VmDetailConstants.ROOT_DISK_CONTROLLER)) { + final String rootDiskController = getOvaDiskControllerDetails(diskTemplate, true); + if (!Strings.isNullOrEmpty(rootDiskController)) { + details.put(VmDetailConstants.ROOT_DISK_CONTROLLER, rootDiskController); + } } } } } + templateVO.setDetails(details); _templateDao.saveDetails(templateVO); return true; @@ -789,7 +799,7 @@ public class TemplateServiceImpl implements TemplateService { String templateName = dataDiskTemplate.isIso() ? dataDiskTemplate.getPath().substring(dataDiskTemplate.getPath().lastIndexOf(File.separator) + 1) : template.getName() + suffix + diskCount; VMTemplateVO templateVO = new VMTemplateVO(templateId, templateName, format, false, false, false, ttype, template.getUrl(), template.requiresHvm(), template.getBits(), template.getAccountId(), null, templateName, false, guestOsId, false, template.getHypervisorType(), null, - null, false, false, false); + null, false, false, false, false); if (dataDiskTemplate.isIso()){ templateVO.setUniqueName(templateName); } @@ -952,7 +962,7 @@ public class TemplateServiceImpl implements TemplateService { AsyncCallFuture future = new AsyncCallFuture(); // no need to create entry on template_store_ref here, since entries are already created when prepareSecondaryStorageForMigration is invoked. // But we need to set default install path so that sync can be done in the right s3 path - TemplateInfo templateOnStore = _templateFactory.getTemplate(template, store); + TemplateInfo templateOnStore = _templateFactory.getTemplate(template, store, null); String installPath = TemplateConstants.DEFAULT_TMPLT_ROOT_DIR + "/" + TemplateConstants.DEFAULT_TMPLT_FIRST_LEVEL_DIR + template.getAccountId() + "/" + template.getId() + "/" + template.getUniqueName(); @@ -1039,7 +1049,7 @@ public class TemplateServiceImpl implements TemplateService { throw new CloudRuntimeException("No secondary VM in running state in source template zone "); } - TemplateObject tmplForCopy = (TemplateObject)_templateFactory.getTemplate(srcTemplate, destStore); + TemplateObject tmplForCopy = (TemplateObject)_templateFactory.getTemplate(srcTemplate, destStore, null); if (s_logger.isDebugEnabled()) { s_logger.debug("Setting source template url to " + url); } @@ -1068,7 +1078,7 @@ public class TemplateServiceImpl implements TemplateService { // clean up already persisted template_store_ref entry in case of createTemplateCallback is never called TemplateDataStoreVO templateStoreVO = _vmTemplateStoreDao.findByStoreTemplate(destStore.getId(), srcTemplate.getId()); if (templateStoreVO != null) { - TemplateInfo tmplObj = _templateFactory.getTemplate(srcTemplate, destStore); + TemplateInfo tmplObj = _templateFactory.getTemplate(srcTemplate, destStore, null); tmplObj.processEvent(ObjectInDataStoreStateMachine.Event.OperationFailed); } TemplateApiResult res = new TemplateApiResult((TemplateObject)templateOnStore); @@ -1130,7 +1140,7 @@ public class TemplateServiceImpl implements TemplateService { @Override public AsyncCallFuture deleteTemplateOnPrimary(TemplateInfo template, StoragePool pool) { - TemplateObject templateObject = (TemplateObject)_templateFactory.getTemplate(template.getId(), (DataStore)pool); + TemplateObject templateObject = (TemplateObject)_templateFactory.getTemplateOnPrimaryStorage(template.getId(), (DataStore)pool, template.getDeployAsIsConfiguration()); templateObject.processEvent(ObjectInDataStoreStateMachine.Event.DestroyRequested); @@ -1241,7 +1251,7 @@ public class TemplateServiceImpl implements TemplateService { dataDiskTemplateOnStore = (TemplateObject)store.create(dataDiskTemplate); dataDiskTemplateOnStore.processEvent(ObjectInDataStoreStateMachine.Event.CreateOnlyRequested); } else { - dataDiskTemplateOnStore = (TemplateObject) imageFactory.getTemplate(parentTemplate, store); + dataDiskTemplateOnStore = (TemplateObject) imageFactory.getTemplate(parentTemplate, store, null); } try { CreateDataDiskTemplateContext context = new CreateDataDiskTemplateContext(null, dataDiskTemplateOnStore, future); @@ -1261,7 +1271,7 @@ public class TemplateServiceImpl implements TemplateService { } protected Void createDatadiskTemplateCallback(AsyncCallbackDispatcher callback, - CreateDataDiskTemplateContext context) { + CreateDataDiskTemplateContext context) { DataObject dataDiskTemplate = context.dataDiskTemplate; AsyncCallFuture future = context.getFuture(); CreateCmdResult result = callback.getResult(); @@ -1280,4 +1290,4 @@ public class TemplateServiceImpl implements TemplateService { future.complete(dataDiskTemplateResult); return null; } -} +} \ No newline at end of file diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/ImageStoreImpl.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/ImageStoreImpl.java index f54673d2b61..d4e2c056763 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/ImageStoreImpl.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/ImageStoreImpl.java @@ -218,8 +218,8 @@ public class ImageStoreImpl implements ImageStoreEntity { } @Override - public List getDataDiskTemplates(DataObject obj) { - return driver.getDataDiskTemplates(obj); + public List getDataDiskTemplates(DataObject obj, String configurationId) { + return driver.getDataDiskTemplates(obj, configurationId); } @Override diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/TemplateObject.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/TemplateObject.java index 86030f226f6..b7a44cd4f08 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/TemplateObject.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/TemplateObject.java @@ -63,6 +63,7 @@ public class TemplateObject implements TemplateInfo { private DataStore dataStore; private String url; private String installPath; // temporarily set installPath before passing to resource for entries with empty installPath for object store migration case + private String deployAsIsConfiguration; // Temporarily set @Inject VMTemplateDao imageDao; @Inject @@ -80,8 +81,9 @@ public class TemplateObject implements TemplateInfo { this.dataStore = dataStore; } - public static TemplateObject getTemplate(VMTemplateVO vo, DataStore store) { + public static TemplateObject getTemplate(VMTemplateVO vo, DataStore store, String configuration) { TemplateObject to = ComponentContext.inject(TemplateObject.class); + to.deployAsIsConfiguration = configuration; to.configure(vo, store); return to; } @@ -190,7 +192,9 @@ public class TemplateObject implements TemplateInfo { if (answer instanceof CopyCmdAnswer) { CopyCmdAnswer cpyAnswer = (CopyCmdAnswer)answer; TemplateObjectTO newTemplate = (TemplateObjectTO)cpyAnswer.getNewData(); - VMTemplateStoragePoolVO templatePoolRef = templatePoolDao.findByPoolTemplate(getDataStore().getId(), getId()); + + String deployAsIsConfiguration = newTemplate.getDeployAsIsConfiguration(); + VMTemplateStoragePoolVO templatePoolRef = templatePoolDao.findByPoolTemplate(getDataStore().getId(), getId(), deployAsIsConfiguration); templatePoolRef.setDownloadPercent(100); setTemplateSizeIfNeeded(newTemplate, templatePoolRef); @@ -327,6 +331,11 @@ public class TemplateObject implements TemplateInfo { return null; } + @Override + public String getDeployAsIsConfiguration() { + return deployAsIsConfiguration; + } + @Override public DataTO getTO() { DataTO to = null; @@ -365,6 +374,14 @@ public class TemplateObject implements TemplateInfo { return this.imageVO.isDirectDownload(); } + @Override + public boolean isDeployAsIs() { + if (this.imageVO == null) { + return false; + } + return this.imageVO.isDeployAsIs(); + } + public void setInstallPath(String installPath) { this.installPath = installPath; } diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/DataObjectManagerImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/DataObjectManagerImpl.java index 2caa3ef3f41..777eb376cb6 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/DataObjectManagerImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/DataObjectManagerImpl.java @@ -76,7 +76,7 @@ public class DataObjectManagerImpl implements DataObjectManager { s_logger.debug("waiting too long for template downloading, marked it as failed"); throw new CloudRuntimeException("waiting too long for template downloading, marked it as failed"); } - return objectInDataStoreMgr.get(dataObj, dataStore); + return objectInDataStoreMgr.get(dataObj, dataStore, null); } class CreateContext extends AsyncRpcContext { diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManager.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManager.java index 8f081d3af39..48acecab6b8 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManager.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManager.java @@ -33,11 +33,11 @@ public interface ObjectInDataStoreManager { boolean deleteIfNotReady(DataObject dataObj); - DataObject get(DataObject dataObj, DataStore store); + DataObject get(DataObject dataObj, DataStore store, String configuration); boolean update(DataObject vo, Event event) throws NoTransitionException, ConcurrentOperationException; - DataObjectInStore findObject(long objId, DataObjectType type, long dataStoreId, DataStoreRole role); + DataObjectInStore findObject(long objId, DataObjectType type, long dataStoreId, DataStoreRole role, String deployAsIsConfiguration); DataObjectInStore findObject(DataObject obj, DataStore store); diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java index ff8112cceff..da97b22946e 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java @@ -118,7 +118,7 @@ public class ObjectInDataStoreManagerImpl implements ObjectInDataStoreManager { public DataObject create(DataObject obj, DataStore dataStore) { if (dataStore.getRole() == DataStoreRole.Primary) { if (obj.getType() == DataObjectType.TEMPLATE) { - VMTemplateStoragePoolVO vo = new VMTemplateStoragePoolVO(dataStore.getId(), obj.getId()); + VMTemplateStoragePoolVO vo = new VMTemplateStoragePoolVO(dataStore.getId(), obj.getId(), null); vo = templatePoolDao.persist(vo); } else if (obj.getType() == DataObjectType.SNAPSHOT) { SnapshotInfo snapshotInfo = (SnapshotInfo)obj; @@ -197,7 +197,7 @@ public class ObjectInDataStoreManagerImpl implements ObjectInDataStoreManager { } } - return this.get(obj, dataStore); + return this.get(obj, dataStore, null); } @Override @@ -206,7 +206,7 @@ public class ObjectInDataStoreManagerImpl implements ObjectInDataStoreManager { DataStore dataStore = dataObj.getDataStore(); if (dataStore.getRole() == DataStoreRole.Primary) { if (dataObj.getType() == DataObjectType.TEMPLATE) { - VMTemplateStoragePoolVO destTmpltPool = templatePoolDao.findByPoolTemplate(dataStore.getId(), objId); + VMTemplateStoragePoolVO destTmpltPool = templatePoolDao.findByPoolTemplate(dataStore.getId(), objId, null); if (destTmpltPool != null) { return templatePoolDao.remove(destTmpltPool.getId()); } else { @@ -254,7 +254,7 @@ public class ObjectInDataStoreManagerImpl implements ObjectInDataStoreManager { DataStore dataStore = dataObj.getDataStore(); if (dataStore.getRole() == DataStoreRole.Primary) { if (dataObj.getType() == DataObjectType.TEMPLATE) { - VMTemplateStoragePoolVO destTmpltPool = templatePoolDao.findByPoolTemplate(dataStore.getId(), objId); + VMTemplateStoragePoolVO destTmpltPool = templatePoolDao.findByPoolTemplate(dataStore.getId(), objId, null); if (destTmpltPool != null && destTmpltPool.getState() != ObjectInDataStoreStateMachine.State.Ready) { return templatePoolDao.remove(destTmpltPool.getId()); } else { @@ -333,9 +333,9 @@ public class ObjectInDataStoreManagerImpl implements ObjectInDataStoreManager { } @Override - public DataObject get(DataObject dataObj, DataStore store) { + public DataObject get(DataObject dataObj, DataStore store, String configuration) { if (dataObj.getType() == DataObjectType.TEMPLATE) { - return imageFactory.getTemplate(dataObj, store); + return imageFactory.getTemplate(dataObj, store, configuration); } else if (dataObj.getType() == DataObjectType.VOLUME) { return volumeFactory.getVolume(dataObj, store); } else if (dataObj.getType() == DataObjectType.SNAPSHOT) { @@ -347,11 +347,15 @@ public class ObjectInDataStoreManagerImpl implements ObjectInDataStoreManager { @Override public DataObjectInStore findObject(DataObject obj, DataStore store) { - return findObject(obj.getId(), obj.getType(), store.getId(), store.getRole()); + String deployAsIsConfiguration = null; + if (obj.getType() == DataObjectType.TEMPLATE) { + deployAsIsConfiguration = ((TemplateInfo) obj).getDeployAsIsConfiguration(); + } + return findObject(obj.getId(), obj.getType(), store.getId(), store.getRole(), deployAsIsConfiguration); } @Override - public DataObjectInStore findObject(long objId, DataObjectType type, long dataStoreId, DataStoreRole role) { + public DataObjectInStore findObject(long objId, DataObjectType type, long dataStoreId, DataStoreRole role, String deployAsIsConfiguration) { DataObjectInStore vo = null; if (role == DataStoreRole.Image || role == DataStoreRole.ImageCache) { switch (type) { @@ -366,7 +370,7 @@ public class ObjectInDataStoreManagerImpl implements ObjectInDataStoreManager { break; } } else if (type == DataObjectType.TEMPLATE && role == DataStoreRole.Primary) { - vo = templatePoolDao.findByPoolTemplate(dataStoreId, objId); + vo = templatePoolDao.findByPoolTemplate(dataStoreId, objId, deployAsIsConfiguration); } else if (type == DataObjectType.SNAPSHOT && role == DataStoreRole.Primary) { vo = snapshotDataStoreDao.findByStoreSnapshot(role, dataStoreId, objId); } else { diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java index 2341603803e..77d97793405 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java @@ -18,6 +18,7 @@ */ package org.apache.cloudstack.storage.image; +import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.sql.PreparedStatement; @@ -32,6 +33,21 @@ import java.util.stream.Collectors; import javax.inject.Inject; +import com.cloud.agent.api.storage.OVFConfigurationTO; +import com.cloud.agent.api.storage.OVFEulaSectionTO; +import com.cloud.agent.api.storage.OVFPropertyTO; +import com.cloud.agent.api.storage.OVFVirtualHardwareItemTO; +import com.cloud.agent.api.storage.OVFVirtualHardwareSectionTO; +import com.cloud.storage.ImageStore; +import com.cloud.storage.Upload; +import com.cloud.storage.VMTemplateDetailVO; +import com.cloud.utils.compression.CompressionUtil; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import org.apache.cloudstack.api.net.NetworkPrerequisiteTO; +import org.apache.commons.collections.CollectionUtils; +import org.apache.log4j.Logger; + import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; @@ -51,8 +67,6 @@ import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao; import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO; import org.apache.cloudstack.storage.endpoint.DefaultEndPointSelector; -import org.apache.commons.collections.CollectionUtils; -import org.apache.log4j.Logger; import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; @@ -60,11 +74,9 @@ import com.cloud.agent.api.storage.CreateDatadiskTemplateCommand; import com.cloud.agent.api.storage.DownloadAnswer; import com.cloud.agent.api.storage.GetDatadisksAnswer; import com.cloud.agent.api.storage.GetDatadisksCommand; -import com.cloud.agent.api.storage.OVFPropertyTO; import com.cloud.agent.api.to.DataObjectType; import com.cloud.agent.api.to.DataTO; import com.cloud.agent.api.to.DatadiskTO; -import com.cloud.agent.api.to.NfsTO; import com.cloud.alert.AlertManager; import com.cloud.configuration.Config; import com.cloud.exception.AgentUnavailableException; @@ -72,14 +84,10 @@ import com.cloud.exception.OperationTimedoutException; import com.cloud.host.dao.HostDao; import com.cloud.secstorage.CommandExecLogDao; import com.cloud.secstorage.CommandExecLogVO; -import com.cloud.storage.DataStoreRole; import com.cloud.storage.StorageManager; -import com.cloud.storage.TemplateOVFPropertyVO; -import com.cloud.storage.Upload; import com.cloud.storage.VMTemplateStorageResourceAssoc; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.VolumeVO; -import com.cloud.storage.dao.TemplateOVFPropertiesDao; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VMTemplateDetailsDao; import com.cloud.storage.dao.VMTemplateZoneDao; @@ -88,14 +96,14 @@ import com.cloud.storage.download.DownloadMonitor; import com.cloud.user.ResourceLimitService; import com.cloud.user.dao.AccountDao; import com.cloud.utils.NumbersUtil; -import com.cloud.utils.crypt.DBEncryptionUtil; import com.cloud.utils.db.TransactionLegacy; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.net.Proxy; import com.cloud.vm.dao.SecondaryStorageVmDao; public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver { - private static final Logger s_logger = Logger.getLogger(BaseImageStoreDriverImpl.class); + private static final Logger LOGGER = Logger.getLogger(BaseImageStoreDriverImpl.class); + @Inject protected VMTemplateDao _templateDao; @Inject @@ -115,7 +123,7 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver { @Inject AlertManager _alertMgr; @Inject - VMTemplateDetailsDao _templateDetailsDao; + VMTemplateDetailsDao templateDetailsDao; @Inject DefaultEndPointSelector _defaultEpSelector; @Inject @@ -123,8 +131,6 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver { @Inject ResourceLimitService _resourceLimitMgr; @Inject - TemplateOVFPropertiesDao templateOvfPropertiesDao; - @Inject HostDao hostDao; @Inject CommandExecLogDao _cmdExecLogDao; @@ -137,6 +143,14 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver { protected String _proxy = null; + private static Gson gson; + + static { + GsonBuilder builder = new GsonBuilder(); + builder.disableHtmlEscaping(); + gson = builder.create(); + } + protected Proxy getHttpProxy() { if (_proxy == null) { return null; @@ -184,14 +198,14 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver { caller.setContext(context); if (data.getType() == DataObjectType.TEMPLATE) { caller.setCallback(caller.getTarget().createTemplateAsyncCallback(null, null)); - if (s_logger.isDebugEnabled()) { - s_logger.debug("Downloading template to data store " + dataStore.getId()); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Downloading template to data store " + dataStore.getId()); } _downloadMonitor.downloadTemplateToStorage(data, caller); } else if (data.getType() == DataObjectType.VOLUME) { caller.setCallback(caller.getTarget().createVolumeAsyncCallback(null, null)); - if (s_logger.isDebugEnabled()) { - s_logger.debug("Downloading volume to data store " + dataStore.getId()); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Downloading volume to data store " + dataStore.getId()); } _downloadMonitor.downloadVolumeToStorage(data, caller); } @@ -201,43 +215,108 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver { * Persist OVF properties as template details for template with id = templateId */ private void persistOVFProperties(List ovfProperties, long templateId) { - List listToPersist = new ArrayList<>(); - for (OVFPropertyTO property : ovfProperties) { - if (!templateOvfPropertiesDao.existsOption(templateId, property.getKey())) { - TemplateOVFPropertyVO option = new TemplateOVFPropertyVO(templateId, property.getKey(), property.getType(), - property.getValue(), property.getQualifiers(), property.isUserConfigurable(), - property.getLabel(), property.getDescription(), property.isPassword()); - if (property.isPassword()) { - String encryptedPassword = DBEncryptionUtil.encrypt(property.getValue()); - option.setValue(encryptedPassword); - } - listToPersist.add(option); - } + if (LOGGER.isTraceEnabled()) { + LOGGER.trace(String.format("saving properties for template %d as details", templateId)); } - if (CollectionUtils.isNotEmpty(listToPersist)) { - s_logger.debug("Persisting " + listToPersist.size() + " OVF properties for template " + templateId); - templateOvfPropertiesDao.saveOptions(listToPersist); + for (OVFPropertyTO property : ovfProperties) { + if (LOGGER.isTraceEnabled()) { + LOGGER.trace(String.format("saving property %s for template %d as detail", property.getKey(), templateId)); + } + persistOvfPropertyAsSetOfTemplateDetails(templateId, property); } } + private void persistOvfPropertyAsSetOfTemplateDetails(long templateId, OVFPropertyTO property) { + String key = property.getKey(); + String propKey = ImageStore.ACS_PROPERTY_PREFIX + key; + try { + String propValue = gson.toJson(property); + savePropertyAttribute(templateId, propKey, propValue); + } catch (RuntimeException re) { + LOGGER.error("gson marshalling of property object fails: " + propKey,re); + } + } + + private void persistNetworkRequirements(List networkRequirements, long templateId) { + if (LOGGER.isTraceEnabled()) { + LOGGER.trace(String.format("saving network requirements for template %d as details", templateId)); + } + for (NetworkPrerequisiteTO network : networkRequirements) { + if (LOGGER.isTraceEnabled()) { + LOGGER.trace(String.format("saving property %s for template %d as detail", network.getName(), templateId)); + } + persistRequiredNetworkAsASingleTemplateDetail(templateId, network); + } + } + + private void persistDiskDefinitions(List disks, long templateId) { + if (LOGGER.isTraceEnabled()) { + LOGGER.trace(String.format("saving disk definitionsn for template %d as details", templateId)); + } + for (DatadiskTO disk : disks) { + if (LOGGER.isTraceEnabled()) { + LOGGER.trace(String.format("saving property %s for template %d as detail", disk.getDiskId(), templateId)); + } + persistDiskDefinitionAsASingleTemplateDetail(templateId, disk); + } + } + + private void persistRequiredNetworkAsASingleTemplateDetail(long templateId, NetworkPrerequisiteTO network) { + String key = network.getName(); + String propKey = ImageStore.REQUIRED_NETWORK_PREFIX + key; + try { + String propValue = gson.toJson(network); + savePropertyAttribute(templateId, propKey, propValue); + } catch (RuntimeException re) { + LOGGER.warn("gson marshalling of network object fails: " + propKey,re); + } + } + + private void persistDiskDefinitionAsASingleTemplateDetail(long templateId, DatadiskTO disk) { + String key = disk.getDiskId(); + String propKey = ImageStore.DISK_DEFINITION_PREFIX + key; + try { + String propValue = gson.toJson(disk); + savePropertyAttribute(templateId, propKey, propValue); + } catch (RuntimeException re) { + LOGGER.warn("gson marshalling of disk definition object fails: " + propKey,re); + } + } + + private void savePropertyAttribute(long templateId, String key, String value) { + if ( templateDetailsDao.findDetail(templateId,key) != null) { + LOGGER.debug(String.format("detail '%s' existed for template %d, deleting.", key, templateId)); + templateDetailsDao.removeDetail(templateId,key); + } + if (LOGGER.isTraceEnabled()) { + LOGGER.trace(String.format("template detail for template %d to save is '%s': '%s'", templateId, key, value)); + } + VMTemplateDetailVO detailVO = new VMTemplateDetailVO(templateId, key, value, false); + LOGGER.debug("Persisting template details " + detailVO.getName() + " from OVF properties for template " + templateId); + templateDetailsDao.persist(detailVO); + } + protected Void createTemplateAsyncCallback(AsyncCallbackDispatcher callback, - CreateContext context) { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Performing image store createTemplate async callback"); + CreateContext context) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Performing image store createTemplate async callback"); } DownloadAnswer answer = callback.getResult(); DataObject obj = context.data; DataStore store = obj.getDataStore(); List ovfProperties = answer.getOvfProperties(); + List networkRequirements = answer.getNetworkRequirements(); + List disks = answer.getDisks(); + OVFVirtualHardwareSectionTO ovfHardwareSection = answer.getOvfHardwareSection(); + List eulaSections = answer.getEulaSections(); + VMTemplateVO template = _templateDao.findById(obj.getId()); TemplateDataStoreVO tmpltStoreVO = _templateStoreDao.findByStoreTemplate(store.getId(), obj.getId()); if (tmpltStoreVO != null) { if (tmpltStoreVO.getDownloadState() == VMTemplateStorageResourceAssoc.Status.DOWNLOADED) { - if (CollectionUtils.isNotEmpty(ovfProperties)) { - persistOVFProperties(ovfProperties, obj.getId()); - } - if (s_logger.isDebugEnabled()) { - s_logger.debug("Template is already in DOWNLOADED state, ignore further incoming DownloadAnswer"); + persistExtraDetails(obj, ovfProperties, networkRequirements, disks, ovfHardwareSection, eulaSections); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Template is already in DOWNLOADED state, ignore further incoming DownloadAnswer"); } return null; } @@ -261,23 +340,21 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver { AsyncCompletionCallback caller = context.getParentCallback(); if (answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR || - answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.ABANDONED || answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.UNKNOWN) { + answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.ABANDONED || answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.UNKNOWN) { CreateCmdResult result = new CreateCmdResult(null, null); result.setSuccess(false); result.setResult(answer.getErrorString()); caller.complete(result); String msg = "Failed to register template: " + obj.getUuid() + " with error: " + answer.getErrorString(); _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_UPLOAD_FAILED, _vmTemplateZoneDao.listByTemplateId(obj.getId()).get(0).getZoneId(), null, msg, msg); - s_logger.error(msg); + LOGGER.error(msg); } else if (answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.DOWNLOADED) { if (answer.getCheckSum() != null) { VMTemplateVO templateDaoBuilder = _templateDao.createForUpdate(); templateDaoBuilder.setChecksum(answer.getCheckSum()); _templateDao.update(obj.getId(), templateDaoBuilder); } - if (CollectionUtils.isNotEmpty(ovfProperties)) { - persistOVFProperties(ovfProperties, obj.getId()); - } + persistExtraDetails(obj, ovfProperties, networkRequirements, disks, ovfHardwareSection, eulaSections); CreateCmdResult result = new CreateCmdResult(null, null); caller.complete(result); @@ -285,8 +362,78 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver { return null; } + private void persistExtraDetails(DataObject obj, List ovfProperties, List networkRequirements, List disks, OVFVirtualHardwareSectionTO ovfHardwareSection, List eulaSections) { + if (LOGGER.isTraceEnabled()) { + LOGGER.trace(String.format("saving %d ovf properties for template '%s' as details", ovfProperties != null ? ovfProperties.size() : 0, obj.getUuid())); + } + if (CollectionUtils.isNotEmpty(ovfProperties)) { + persistOVFProperties(ovfProperties, obj.getId()); + } + if (LOGGER.isTraceEnabled()) { + LOGGER.trace(String.format("saving %d required network requirements for template '%s' as details", networkRequirements != null ? networkRequirements.size() : 0, obj.getUuid())); + } + if (CollectionUtils.isNotEmpty(networkRequirements)) { + persistNetworkRequirements(networkRequirements, obj.getId()); + } + if (LOGGER.isTraceEnabled()) { + LOGGER.trace(String.format("saving %d disks definitions for template '%s' as details", disks != null ? disks.size() : 0, obj.getUuid())); + } + if (CollectionUtils.isNotEmpty(disks)) { + persistDiskDefinitions(disks, obj.getId()); + } + if (CollectionUtils.isNotEmpty(eulaSections)) { + persistEulaSectionsAsTemplateDetails(eulaSections, obj.getId()); + } + persistOVFHardwareSectionAsTemplateDetails(ovfHardwareSection, obj.getId()); + } + + private void persistEulaSectionsAsTemplateDetails(List eulaSections, long templateId) { + CompressionUtil compressionUtil = new CompressionUtil(); + for (OVFEulaSectionTO eulaSectionTO : eulaSections) { + String key = ImageStore.OVF_EULA_SECTION_PREFIX + eulaSectionTO.getIndex() + "-" + eulaSectionTO.getInfo(); + byte[] compressedLicense = eulaSectionTO.getCompressedLicense(); + try { + String detailValue = compressionUtil.decompressByteArary(compressedLicense); + savePropertyAttribute(templateId, key, detailValue); + } catch (IOException e) { + LOGGER.error("Could not decompress the license for template " + templateId, e); + } + } + } + + /** + * Persist template details for template with ID=templateId, with name=key and value=json(object) + */ + private void persistTemplateDetailGsonEncoded(long templateId, String key, Object object) { + try { + String propValue = gson.toJson(object); + savePropertyAttribute(templateId, key, propValue); + } catch (RuntimeException re) { + LOGGER.error("gson marshalling of property object fails: " + key, re); + } + } + + private void persistOVFHardwareSectionAsTemplateDetails(OVFVirtualHardwareSectionTO ovfHardwareSection, long templateId) { + if (ovfHardwareSection != null) { + if (CollectionUtils.isNotEmpty(ovfHardwareSection.getConfigurations())) { + for (OVFConfigurationTO configuration : ovfHardwareSection.getConfigurations()) { + String key = configuration.getId(); + String propKey = ImageStore.OVF_HARDWARE_CONFIGURATION_PREFIX + configuration.getIndex() + "-" + key; + persistTemplateDetailGsonEncoded(templateId, propKey, configuration); + } + } + if (CollectionUtils.isNotEmpty(ovfHardwareSection.getCommonHardwareItems())) { + for (OVFVirtualHardwareItemTO item : ovfHardwareSection.getCommonHardwareItems()) { + String key = item.getResourceType().getName().trim().replaceAll("\\s","") + "-" + item.getInstanceId(); + String propKey = ImageStore.OVF_HARDWARE_ITEM_PREFIX + key; + persistTemplateDetailGsonEncoded(templateId, propKey, item); + } + } + } + } + protected Void - createVolumeAsyncCallback(AsyncCallbackDispatcher callback, CreateContext context) { + createVolumeAsyncCallback(AsyncCallbackDispatcher callback, CreateContext context) { DownloadAnswer answer = callback.getResult(); DataObject obj = context.data; DataStore store = obj.getDataStore(); @@ -294,8 +441,8 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver { VolumeDataStoreVO volStoreVO = _volumeStoreDao.findByStoreVolume(store.getId(), obj.getId()); if (volStoreVO != null) { if (volStoreVO.getDownloadState() == VMTemplateStorageResourceAssoc.Status.DOWNLOADED) { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Volume is already in DOWNLOADED state, ignore further incoming DownloadAnswer"); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Volume is already in DOWNLOADED state, ignore further incoming DownloadAnswer"); } return null; } @@ -319,7 +466,7 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver { AsyncCompletionCallback caller = context.getParentCallback(); if (answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR || - answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.ABANDONED || answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.UNKNOWN) { + answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.ABANDONED || answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.UNKNOWN) { CreateCmdResult result = new CreateCmdResult(null, null); result.setSuccess(false); result.setResult(answer.getErrorString()); @@ -327,7 +474,7 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver { String msg = "Failed to upload volume: " + obj.getUuid() + " with error: " + answer.getErrorString(); _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_UPLOAD_FAILED, (volStoreVO == null ? -1L : volStoreVO.getZoneId()), null, msg, msg); - s_logger.error(msg); + LOGGER.error(msg); } else if (answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.DOWNLOADED) { CreateCmdResult result = new CreateCmdResult(null, null); caller.complete(result); @@ -344,7 +491,7 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver { Answer answer = null; if (ep == null) { String errMsg = "No remote endpoint to send command, check if host or ssvm is down?"; - s_logger.error(errMsg); + LOGGER.error(errMsg); answer = new Answer(cmd, false, errMsg); } else { answer = ep.sendMessage(cmd); @@ -353,7 +500,7 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver { result.setResult(answer.getDetails()); } } catch (Exception ex) { - s_logger.debug("Unable to destoy " + data.getType().toString() + ": " + data.getId(), ex); + LOGGER.debug("Unable to destoy " + data.getType().toString() + ": " + data.getId(), ex); result.setResult(ex.toString()); } callback.complete(result); @@ -377,7 +524,7 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver { List eps = _epSelector.findAllEndpointsForScope(srcdata.getDataStore()); if (eps == null || eps.isEmpty()) { String errMsg = "No remote endpoint to send command, check if host or ssvm is down?"; - s_logger.error(errMsg); + LOGGER.error(errMsg); answer = new Answer(cmd, false, errMsg); } else { // select endpoint with least number of commands running on them @@ -414,25 +561,16 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver { return answer; } catch (AgentUnavailableException e) { errMsg = e.toString(); - s_logger.debug("Failed to send command, due to Agent:" + endPoint.getId() + ", " + e.toString()); + LOGGER.debug("Failed to send command, due to Agent:" + endPoint.getId() + ", " + e.toString()); } catch (OperationTimedoutException e) { errMsg = e.toString(); - s_logger.debug("Failed to send command, due to Agent:" + endPoint.getId() + ", " + e.toString()); + LOGGER.debug("Failed to send command, due to Agent:" + endPoint.getId() + ", " + e.toString()); } throw new CloudRuntimeException("Failed to send command, due to Agent:" + endPoint.getId() + ", " + errMsg); } @Override public boolean canCopy(DataObject srcData, DataObject destData) { - DataStore srcStore = srcData.getDataStore(); - DataStore destStore = destData.getDataStore(); - if ((srcData.getDataStore().getTO() instanceof NfsTO && destData.getDataStore().getTO() instanceof NfsTO) && - (srcStore.getRole() == DataStoreRole.Image && destStore.getRole() == DataStoreRole.Image) && - ((srcData.getType() == DataObjectType.TEMPLATE && destData.getType() == DataObjectType.TEMPLATE) || - (srcData.getType() == DataObjectType.SNAPSHOT && destData.getType() == DataObjectType.SNAPSHOT) || - (srcData.getType() == DataObjectType.VOLUME && destData.getType() == DataObjectType.VOLUME))) { - return true; - } return false; } @@ -445,18 +583,18 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver { } @Override - public List getDataDiskTemplates(DataObject obj) { + public List getDataDiskTemplates(DataObject obj, String configurationId) { List dataDiskDetails = new ArrayList(); - if (s_logger.isDebugEnabled()) { - s_logger.debug("Get the data disks present in the OVA template"); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Get the data disks present in the OVA template"); } DataStore store = obj.getDataStore(); - GetDatadisksCommand cmd = new GetDatadisksCommand(obj.getTO()); + GetDatadisksCommand cmd = new GetDatadisksCommand(obj.getTO(), configurationId); EndPoint ep = _defaultEpSelector.select(store); Answer answer = null; if (ep == null) { String errMsg = "No remote endpoint to send command, check if host or ssvm is down?"; - s_logger.error(errMsg); + LOGGER.error(errMsg); answer = new Answer(cmd, false, errMsg); } else { answer = ep.sendMessage(cmd); @@ -475,14 +613,14 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver { public Void createDataDiskTemplateAsync(TemplateInfo dataDiskTemplate, String path, String diskId, boolean bootable, long fileSize, AsyncCompletionCallback callback) { Answer answer = null; String errMsg = null; - if (s_logger.isDebugEnabled()) { - s_logger.debug("Create Datadisk template: " + dataDiskTemplate.getId()); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Create Datadisk template: " + dataDiskTemplate.getId()); } CreateDatadiskTemplateCommand cmd = new CreateDatadiskTemplateCommand(dataDiskTemplate.getTO(), path, diskId, fileSize, bootable); EndPoint ep = _defaultEpSelector.select(dataDiskTemplate.getDataStore()); if (ep == null) { errMsg = "No remote endpoint to send command, check if host or ssvm is down?"; - s_logger.error(errMsg); + LOGGER.error(errMsg); answer = new Answer(cmd, false, errMsg); } else { answer = ep.sendMessage(cmd); @@ -496,8 +634,12 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver { return null; } + private Integer getCopyCmdsCountToSpecificSSVM(Long ssvmId) { + return _cmdExecLogDao.getCopyCmdCountForSSVM(ssvmId); + } + private List ssvmWithLeastMigrateJobs() { - s_logger.debug("Picking ssvm from the pool with least commands running on it"); + LOGGER.debug("Picking ssvm from the pool with least commands running on it"); String query = "select host_id, count(*) from cmd_exec_log group by host_id order by 2;"; TransactionLegacy txn = TransactionLegacy.currentTxn(); @@ -510,7 +652,7 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver { result.add((long) rs.getInt(1)); } } catch (SQLException e) { - s_logger.debug("SQLException caught", e); + LOGGER.debug("SQLException caught", e); } return result; } diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/ImageStoreDriver.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/ImageStoreDriver.java index 70f40f6f5c0..29071d8e5bb 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/ImageStoreDriver.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/ImageStoreDriver.java @@ -37,7 +37,7 @@ public interface ImageStoreDriver extends DataStoreDriver { void deleteEntityExtractUrl(DataStore store, String installPath, String url, Upload.Type entityType); - List getDataDiskTemplates(DataObject obj); + List getDataDiskTemplates(DataObject obj, String configurationId); Void createDataDiskTemplateAsync(TemplateInfo dataDiskTemplate, String path, String diskId, boolean bootable, long fileSize, AsyncCompletionCallback callback); } diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/TemplateEntityImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/TemplateEntityImpl.java deleted file mode 100644 index b027c42a86e..00000000000 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/TemplateEntityImpl.java +++ /dev/null @@ -1,314 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.cloudstack.storage.image; - -import java.lang.reflect.Method; -import java.util.Date; -import java.util.List; -import java.util.Map; - -import org.apache.cloudstack.engine.cloud.entity.api.TemplateEntity; -import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; -import org.apache.cloudstack.storage.image.datastore.ImageStoreInfo; - -import com.cloud.hypervisor.Hypervisor.HypervisorType; -import com.cloud.storage.Storage.ImageFormat; -import com.cloud.storage.Storage.TemplateType; -import com.cloud.template.VirtualMachineTemplate; - -public class TemplateEntityImpl implements TemplateEntity { - protected TemplateInfo templateInfo; - - @Override - public State getState() { - return templateInfo.getState(); - } - - public TemplateEntityImpl(TemplateInfo templateInfo) { - this.templateInfo = templateInfo; - } - - public ImageStoreInfo getImageDataStore() { - return (ImageStoreInfo)templateInfo.getDataStore(); - } - - public long getImageDataStoreId() { - return getImageDataStore().getImageStoreId(); - } - - public TemplateInfo getTemplateInfo() { - return templateInfo; - } - - @Override - public String getUuid() { - return templateInfo.getUuid(); - } - - @Override - public long getId() { - return templateInfo.getId(); - } - - public String getExternalId() { - // TODO Auto-generated method stub - return null; - } - - @Override - public String getCurrentState() { - // TODO Auto-generated method stub - return null; - } - - @Override - public String getDesiredState() { - // TODO Auto-generated method stub - return null; - } - - @Override - public Date getCreatedTime() { - // TODO Auto-generated method stub - return null; - } - - @Override - public Date getLastUpdatedTime() { - // TODO Auto-generated method stub - return null; - } - - @Override - public String getOwner() { - // TODO Auto-generated method stub - return null; - } - - @Override - public Map getDetails() { - // TODO Auto-generated method stub - return null; - } - - @Override - public boolean isDynamicallyScalable() { - return false; - } - - @Override - public void addDetail(String name, String value) { - // TODO Auto-generated method stub - - } - - @Override - public void delDetail(String name, String value) { - // TODO Auto-generated method stub - - } - - @Override - public void updateDetail(String name, String value) { - // TODO Auto-generated method stub - - } - - @Override - public List getApplicableActions() { - // TODO Auto-generated method stub - return null; - } - - @Override - public boolean isFeatured() { - // TODO Auto-generated method stub - return false; - } - - @Override - public boolean isPublicTemplate() { - // TODO Auto-generated method stub - return false; - } - - @Override - public boolean isExtractable() { - // TODO Auto-generated method stub - return false; - } - - @Override - public String getName() { - // TODO Auto-generated method stub - return null; - } - - @Override - public ImageFormat getFormat() { - // TODO Auto-generated method stub - return null; - } - - @Override - public boolean isRequiresHvm() { - // TODO Auto-generated method stub - return false; - } - - @Override - public String getDisplayText() { - // TODO Auto-generated method stub - return null; - } - - @Override - public boolean isEnablePassword() { - // TODO Auto-generated method stub - return false; - } - - @Override - public boolean isEnableSshKey() { - // TODO Auto-generated method stub - return false; - } - - @Override - public boolean isCrossZones() { - // TODO Auto-generated method stub - return false; - } - - @Override - public Date getCreated() { - // TODO Auto-generated method stub - return null; - } - - @Override - public long getGuestOSId() { - // TODO Auto-generated method stub - return 0; - } - - @Override - public boolean isBootable() { - // TODO Auto-generated method stub - return false; - } - - @Override - public TemplateType getTemplateType() { - // TODO Auto-generated method stub - return null; - } - - @Override - public HypervisorType getHypervisorType() { - // TODO Auto-generated method stub - return null; - } - - @Override - public int getBits() { - // TODO Auto-generated method stub - return 0; - } - - @Override - public String getUniqueName() { - // TODO Auto-generated method stub - return null; - } - - @Override - public String getUrl() { - // TODO Auto-generated method stub - return null; - } - - @Override - public String getChecksum() { - // TODO Auto-generated method stub - return null; - } - - @Override - public Long getSourceTemplateId() { - // TODO Auto-generated method stub - return null; - } - - @Override - public String getTemplateTag() { - // TODO Auto-generated method stub - return null; - } - - @Override - public long getAccountId() { - // TODO Auto-generated method stub - return 0; - } - - @Override - public long getDomainId() { - // TODO Auto-generated method stub - return 0; - } - - @Override - public long getPhysicalSize() { - // TODO Auto-generated method stub - return 0; - } - - @Override - public long getVirtualSize() { - // TODO Auto-generated method stub - return 0; - } - - @Override - public Class getEntityType() { - return VirtualMachineTemplate.class; - } - - @Override - public long getUpdatedCount() { - // TODO Auto-generated method stub - return 0; - } - - @Override - public void incrUpdatedCount() { - // TODO Auto-generated method stub - } - - @Override - public Date getUpdated() { - return null; - } - - @Override - public Long getParentTemplateId() { - return null; - } -} diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java index 16c3396884e..18a7f3c4890 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java @@ -218,12 +218,12 @@ public class PrimaryDataStoreImpl implements PrimaryDataStore { } @Override - public TemplateInfo getTemplate(long templateId) { - VMTemplateStoragePoolVO template = templatePoolDao.findByPoolTemplate(getId(), templateId); + public TemplateInfo getTemplate(long templateId, String configuration) { + VMTemplateStoragePoolVO template = templatePoolDao.findByPoolTemplate(getId(), templateId, configuration); if (template == null || template.getState() != ObjectInDataStoreStateMachine.State.Ready) { return null; } - return imageDataFactory.getTemplate(templateId, this); + return imageDataFactory.getTemplateOnPrimaryStorage(templateId, this, configuration); } @Override @@ -264,18 +264,26 @@ public class PrimaryDataStoreImpl implements PrimaryDataStore { */ @Override public DataObject create(DataObject dataObject) { - return create(dataObject, true); + return create(dataObject, true, null); + } + + @Override + public DataObject create(DataObject dataObject, String configuration) { + return create(dataObject, true, configuration); } /** * Please read the comment for the create(DataObject) method if you are planning on passing in "false" for createEntryInTempSpoolRef. + * + * The parameter configuration allows storing multiple configurations of the + * base template appliance in primary storage (VMware supported) - null by default or no configurations */ @Override - public DataObject create(DataObject obj, boolean createEntryInTempSpoolRef) { + public DataObject create(DataObject obj, boolean createEntryInTempSpoolRef, String configuration) { // create template on primary storage if (obj.getType() == DataObjectType.TEMPLATE && (!isManaged() || (createEntryInTempSpoolRef && canCloneVolume()))) { try { - String templateIdPoolIdString = "templateId:" + obj.getId() + "poolId:" + getId(); + String templateIdPoolIdString = "templateId:" + obj.getId() + "poolId:" + getId() + "conf:" + configuration; VMTemplateStoragePoolVO templateStoragePoolRef; GlobalLock lock = GlobalLock.getInternLock(templateIdPoolIdString); if (!lock.lock(5)) { @@ -283,20 +291,20 @@ public class PrimaryDataStoreImpl implements PrimaryDataStore { return null; } try { - templateStoragePoolRef = templatePoolDao.findByPoolTemplate(getId(), obj.getId()); + templateStoragePoolRef = templatePoolDao.findByPoolTemplate(getId(), obj.getId(), configuration); if (templateStoragePoolRef == null) { if (s_logger.isDebugEnabled()) { s_logger.debug("Not found (" + templateIdPoolIdString + ") in template_spool_ref, persisting it"); } - templateStoragePoolRef = new VMTemplateStoragePoolVO(getId(), obj.getId()); + templateStoragePoolRef = new VMTemplateStoragePoolVO(getId(), obj.getId(), configuration); templateStoragePoolRef = templatePoolDao.persist(templateStoragePoolRef); } } catch (Throwable t) { if (s_logger.isDebugEnabled()) { s_logger.debug("Failed to insert (" + templateIdPoolIdString + ") to template_spool_ref", t); } - templateStoragePoolRef = templatePoolDao.findByPoolTemplate(getId(), obj.getId()); + templateStoragePoolRef = templatePoolDao.findByPoolTemplate(getId(), obj.getId(), configuration); if (templateStoragePoolRef == null) { throw new CloudRuntimeException("Failed to create template storage pool entry"); } else { @@ -321,7 +329,7 @@ public class PrimaryDataStoreImpl implements PrimaryDataStore { } } - return objectInStoreMgr.get(obj, this); + return objectInStoreMgr.get(obj, this, configuration); } @Override diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java index 76e59d82856..d2a771d46a5 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java @@ -20,6 +20,14 @@ import java.util.Date; import javax.inject.Inject; +import com.cloud.storage.MigrationOptions; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.VolumeDetailVO; +import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.storage.dao.VolumeDetailsDao; +import com.cloud.vm.VmDetailConstants; +import org.apache.log4j.Logger; + import org.apache.cloudstack.engine.subsystem.api.storage.DataObjectInStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; @@ -30,7 +38,6 @@ import org.apache.cloudstack.storage.datastore.ObjectInDataStoreManager; import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao; import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO; import org.apache.cloudstack.storage.to.VolumeObjectTO; -import org.apache.log4j.Logger; import com.cloud.agent.api.Answer; import com.cloud.agent.api.storage.DownloadAnswer; @@ -71,6 +78,10 @@ public class VolumeObject implements VolumeInfo { VMInstanceDao vmInstanceDao; @Inject DiskOfferingDao diskOfferingDao; + @Inject + VMTemplateDao templateDao; + @Inject + VolumeDetailsDao volumeDetailsDao; private Object payload; private MigrationOptions migrationOptions; private boolean directDownload; @@ -357,7 +368,7 @@ public class VolumeObject implements VolumeInfo { if (dataStore == null) { throw new CloudRuntimeException("datastore must be set before using this object"); } - DataObjectInStore obj = objectInStoreMgr.findObject(volumeVO.getId(), DataObjectType.VOLUME, dataStore.getId(), dataStore.getRole()); + DataObjectInStore obj = objectInStoreMgr.findObject(volumeVO.getId(), DataObjectType.VOLUME, dataStore.getId(), dataStore.getRole(), null); if (obj.getState() != ObjectInDataStoreStateMachine.State.Ready) { return dataStore.getUri() + "&" + EncodingType.OBJTYPE + "=" + DataObjectType.VOLUME + "&" + EncodingType.SIZE + "=" + volumeVO.getSize() + "&" + EncodingType.NAME + "=" + volumeVO.getName(); @@ -391,9 +402,7 @@ public class VolumeObject implements VolumeInfo { if (event == ObjectInDataStoreStateMachine.Event.CreateOnlyRequested) { volEvent = Volume.Event.UploadRequested; } else if (event == ObjectInDataStoreStateMachine.Event.MigrationRequested) { - volEvent = Event.CopyRequested; - } else if (event == ObjectInDataStoreStateMachine.Event.MigrateDataRequested) { - return; + volEvent = Volume.Event.CopyRequested; } } else { if (event == ObjectInDataStoreStateMachine.Event.CreateRequested || event == ObjectInDataStoreStateMachine.Event.CreateOnlyRequested) { @@ -436,6 +445,18 @@ public class VolumeObject implements VolumeInfo { } } + @Override + public boolean isDeployAsIs() { + VMTemplateVO template = templateDao.findById(getTemplateId()); + return template != null && template.isDeployAsIs(); + } + + @Override + public String getDeployAsIsConfiguration() { + VolumeDetailVO detail = volumeDetailsDao.findDetail(getId(), VmDetailConstants.DEPLOY_AS_IS_CONFIGURATION); + return detail != null ? detail.getValue() : null; + } + @Override public void processEventOnly(ObjectInDataStoreStateMachine.Event event) { try { diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java index 77413ad6c2b..3ccb7be8f60 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java @@ -121,6 +121,8 @@ import com.cloud.utils.db.GlobalLock; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VirtualMachine; +import static com.cloud.storage.resource.StorageProcessor.REQUEST_TEMPLATE_RELOAD; + @Component public class VolumeServiceImpl implements VolumeService { private static final Logger s_logger = Logger.getLogger(VolumeServiceImpl.class); @@ -458,7 +460,7 @@ public class VolumeServiceImpl implements VolumeService { private final AsyncCallFuture _future; public ManagedCreateBaseImageContext(AsyncCompletionCallback callback, VolumeInfo volumeInfo, PrimaryDataStore primaryDatastore, TemplateInfo templateInfo, - AsyncCallFuture future) { + AsyncCallFuture future) { super(callback); _volumeInfo = volumeInfo; @@ -493,7 +495,7 @@ public class VolumeServiceImpl implements VolumeService { long templatePoolId; public CreateBaseImageContext(AsyncCompletionCallback callback, VolumeInfo volume, PrimaryDataStore datastore, TemplateInfo srcTemplate, AsyncCallFuture future, - DataObject destObj, long templatePoolId) { + DataObject destObj, long templatePoolId) { super(callback); this.volume = volume; this.dataStore = datastore; @@ -530,7 +532,7 @@ public class VolumeServiceImpl implements VolumeService { int sleepTime = 120; int tries = storagePoolMaxWaitSeconds / sleepTime; while (tries > 0) { - TemplateInfo tmpl = store.getTemplate(template.getId()); + TemplateInfo tmpl = store.getTemplate(template.getId(), null); if (tmpl != null) { return tmpl; } @@ -546,9 +548,10 @@ public class VolumeServiceImpl implements VolumeService { @DB protected void createBaseImageAsync(VolumeInfo volume, PrimaryDataStore dataStore, TemplateInfo template, AsyncCallFuture future) { - DataObject templateOnPrimaryStoreObj = dataStore.create(template); + String deployAsIsConfiguration = volume.getDeployAsIsConfiguration(); + DataObject templateOnPrimaryStoreObj = dataStore.create(template, deployAsIsConfiguration); - VMTemplateStoragePoolVO templatePoolRef = _tmpltPoolDao.findByPoolTemplate(dataStore.getId(), template.getId()); + VMTemplateStoragePoolVO templatePoolRef = _tmpltPoolDao.findByPoolTemplate(dataStore.getId(), template.getId(), deployAsIsConfiguration); if (templatePoolRef == null) { throw new CloudRuntimeException("Failed to find template " + template.getUniqueName() + " in storage pool " + dataStore.getId()); } else { @@ -571,8 +574,8 @@ public class VolumeServiceImpl implements VolumeService { if (s_logger.isDebugEnabled()) { s_logger.info("Unable to acquire lock on VMTemplateStoragePool " + templatePoolRefId); } - templatePoolRef = _tmpltPoolDao.findByPoolTemplate(dataStore.getId(), template.getId()); - if (templatePoolRef != null && templatePoolRef.getState() == ObjectInDataStoreStateMachine.State.Ready) { + templatePoolRef = _tmpltPoolDao.findByPoolTemplate(dataStore.getId(), template.getId(), deployAsIsConfiguration); + if (templatePoolRef != null && templatePoolRef.getState() == ObjectInDataStoreStateMachine.State.Ready && !template.isDeployAsIs()) { s_logger.info( "Unable to acquire lock on VMTemplateStoragePool " + templatePoolRefId + ", But Template " + template.getUniqueName() + " is already copied to primary storage, skip copying"); createVolumeFromBaseImageAsync(volume, templateOnPrimaryStoreObj, dataStore, future); @@ -585,7 +588,7 @@ public class VolumeServiceImpl implements VolumeService { s_logger.info("lock is acquired for VMTemplateStoragePool " + templatePoolRefId); } try { - if (templatePoolRef.getState() == ObjectInDataStoreStateMachine.State.Ready) { + if (templatePoolRef.getState() == ObjectInDataStoreStateMachine.State.Ready && !template.isDeployAsIs()) { s_logger.info("Template " + template.getUniqueName() + " is already copied to primary storage, skip copying"); createVolumeFromBaseImageAsync(volume, templateOnPrimaryStoreObj, dataStore, future); return; @@ -595,7 +598,7 @@ public class VolumeServiceImpl implements VolumeService { } catch (Throwable e) { s_logger.debug("failed to create template on storage", e); templateOnPrimaryStoreObj.processEvent(Event.OperationFailed); - dataStore.create(template); // make sure that template_spool_ref entry is still present so that the second thread can acquire the lock + dataStore.create(template, deployAsIsConfiguration); // make sure that template_spool_ref entry is still present so that the second thread can acquire the lock VolumeApiResult result = new VolumeApiResult(volume); result.setResult(e.toString()); future.complete(result); @@ -705,14 +708,16 @@ public class VolumeServiceImpl implements VolumeService { private final AsyncCallFuture future; private final DataObject templateOnStore; private final SnapshotInfo snapshot; + private final String deployAsIsConfiguration; public CreateVolumeFromBaseImageContext(AsyncCompletionCallback callback, DataObject vo, DataStore primaryStore, DataObject templateOnStore, AsyncCallFuture future, - SnapshotInfo snapshot) { + SnapshotInfo snapshot, String configuration) { super(callback); this.vo = vo; this.future = future; this.templateOnStore = templateOnStore; this.snapshot = snapshot; + this.deployAsIsConfiguration = configuration; } public AsyncCallFuture getFuture() { @@ -722,15 +727,16 @@ public class VolumeServiceImpl implements VolumeService { @DB protected void createVolumeFromBaseImageAsync(VolumeInfo volume, DataObject templateOnPrimaryStore, PrimaryDataStore pd, AsyncCallFuture future) { - DataObject volumeOnPrimaryStorage = pd.create(volume); + DataObject volumeOnPrimaryStorage = pd.create(volume, volume.getDeployAsIsConfiguration()); volumeOnPrimaryStorage.processEvent(Event.CreateOnlyRequested); - CreateVolumeFromBaseImageContext context = new CreateVolumeFromBaseImageContext(null, volumeOnPrimaryStorage, pd, templateOnPrimaryStore, future, null); + CreateVolumeFromBaseImageContext context = new CreateVolumeFromBaseImageContext(null, volumeOnPrimaryStorage, pd, templateOnPrimaryStore, future, null, volume.getDeployAsIsConfiguration()); AsyncCallbackDispatcher caller = AsyncCallbackDispatcher.create(this); caller.setCallback(caller.getTarget().createVolumeFromBaseImageCallBack(null, null)); caller.setContext(context); motionSrv.copyAsync(context.templateOnStore, volumeOnPrimaryStorage, caller); + return; } @@ -740,6 +746,7 @@ public class VolumeServiceImpl implements VolumeService { DataObject tmplOnPrimary = context.templateOnStore; CopyCommandResult result = callback.getResult(); VolumeApiResult volResult = new VolumeApiResult((VolumeObject)vo); + String deployAsIsConfiguration = context.deployAsIsConfiguration; if (result.isSuccess()) { vo.processEvent(Event.OperationSuccessed, result.getAnswer()); @@ -750,10 +757,10 @@ public class VolumeServiceImpl implements VolumeService { // hack for Vmware: host is down, previously download template to the host needs to be re-downloaded, so we need to reset // template_spool_ref entry here to NOT_DOWNLOADED and Allocated state Answer ans = result.getAnswer(); - if (ans != null && ans instanceof CopyCmdAnswer && ans.getDetails().contains("request template reload")) { + if (ans != null && ans instanceof CopyCmdAnswer && ans.getDetails().contains(REQUEST_TEMPLATE_RELOAD)) { if (tmplOnPrimary != null) { s_logger.info("Reset template_spool_ref entry so that vmware template can be reloaded in next try"); - VMTemplateStoragePoolVO templatePoolRef = _tmpltPoolDao.findByPoolTemplate(tmplOnPrimary.getDataStore().getId(), tmplOnPrimary.getId()); + VMTemplateStoragePoolVO templatePoolRef = _tmpltPoolDao.findByPoolTemplate(tmplOnPrimary.getDataStore().getId(), tmplOnPrimary.getId(), deployAsIsConfiguration); if (templatePoolRef != null) { long templatePoolRefId = templatePoolRef.getId(); templatePoolRef = _tmpltPoolDao.acquireInLockTable(templatePoolRefId, 1200); @@ -789,9 +796,9 @@ public class VolumeServiceImpl implements VolumeService { private TemplateInfo createManagedTemplateVolume(TemplateInfo srcTemplateInfo, PrimaryDataStore destPrimaryDataStore) { // create a template volume on primary storage AsyncCallFuture createTemplateFuture = new AsyncCallFuture<>(); - TemplateInfo templateOnPrimary = (TemplateInfo)destPrimaryDataStore.create(srcTemplateInfo); + TemplateInfo templateOnPrimary = (TemplateInfo)destPrimaryDataStore.create(srcTemplateInfo, srcTemplateInfo.getDeployAsIsConfiguration()); - VMTemplateStoragePoolVO templatePoolRef = _tmpltPoolDao.findByPoolTemplate(destPrimaryDataStore.getId(), templateOnPrimary.getId()); + VMTemplateStoragePoolVO templatePoolRef = _tmpltPoolDao.findByPoolTemplate(destPrimaryDataStore.getId(), templateOnPrimary.getId(), srcTemplateInfo.getDeployAsIsConfiguration()); if (templatePoolRef == null) { throw new CloudRuntimeException("Failed to find template " + srcTemplateInfo.getUniqueName() + " in storage pool " + destPrimaryDataStore.getId()); @@ -861,7 +868,7 @@ public class VolumeServiceImpl implements VolumeService { * @param destHost The host that we will use for the copy */ private void copyTemplateToManagedTemplateVolume(TemplateInfo srcTemplateInfo, TemplateInfo templateOnPrimary, VMTemplateStoragePoolVO templatePoolRef, PrimaryDataStore destPrimaryDataStore, - Host destHost) { + Host destHost) { AsyncCallFuture copyTemplateFuture = new AsyncCallFuture<>(); int storagePoolMaxWaitSeconds = NumbersUtil.parseInt(configDao.getValue(Config.StoragePoolMaxWaitSeconds.key()), 3600); long templatePoolRefId = templatePoolRef.getId(); @@ -986,7 +993,7 @@ public class VolumeServiceImpl implements VolumeService { * @param future For async */ private void createManagedVolumeCloneTemplateAsync(VolumeInfo volumeInfo, TemplateInfo templateOnPrimary, PrimaryDataStore destPrimaryDataStore, AsyncCallFuture future) { - VMTemplateStoragePoolVO templatePoolRef = _tmpltPoolDao.findByPoolTemplate(destPrimaryDataStore.getId(), templateOnPrimary.getId()); + VMTemplateStoragePoolVO templatePoolRef = _tmpltPoolDao.findByPoolTemplate(destPrimaryDataStore.getId(), templateOnPrimary.getId(), volumeInfo.getDeployAsIsConfiguration()); if (templatePoolRef == null) { throw new CloudRuntimeException("Failed to find template " + templateOnPrimary.getUniqueName() + " in storage pool " + destPrimaryDataStore.getId()); @@ -1000,7 +1007,7 @@ public class VolumeServiceImpl implements VolumeService { try { volumeInfo.processEvent(Event.CreateOnlyRequested); - CreateVolumeFromBaseImageContext context = new CreateVolumeFromBaseImageContext<>(null, volumeInfo, destPrimaryDataStore, templateOnPrimary, future, null); + CreateVolumeFromBaseImageContext context = new CreateVolumeFromBaseImageContext<>(null, volumeInfo, destPrimaryDataStore, templateOnPrimary, future, null, volumeInfo.getDeployAsIsConfiguration()); AsyncCallbackDispatcher caller = AsyncCallbackDispatcher.create(this); @@ -1021,7 +1028,7 @@ public class VolumeServiceImpl implements VolumeService { try { // Create a volume on managed storage. - TemplateInfo destTemplateInfo = (TemplateInfo)primaryDataStore.create(srcTemplateInfo, false); + TemplateInfo destTemplateInfo = (TemplateInfo)primaryDataStore.create(srcTemplateInfo, false, volumeInfo.getDeployAsIsConfiguration()); AsyncCallFuture createVolumeFuture = createVolumeAsync(volumeInfo, primaryDataStore); VolumeApiResult createVolumeResult = createVolumeFuture.get(); @@ -1109,7 +1116,7 @@ public class VolumeServiceImpl implements VolumeService { if (storageCanCloneVolume && computeSupportsVolumeClone) { s_logger.debug("Storage " + destDataStoreId + " can support cloning using a cached template and compute side is OK with volume cloning."); - TemplateInfo templateOnPrimary = destPrimaryDataStore.getTemplate(srcTemplateInfo.getId()); + TemplateInfo templateOnPrimary = destPrimaryDataStore.getTemplate(srcTemplateInfo.getId(), null); if (templateOnPrimary == null) { templateOnPrimary = createManagedTemplateVolume(srcTemplateInfo, destPrimaryDataStore); @@ -1120,7 +1127,7 @@ public class VolumeServiceImpl implements VolumeService { } // Copy the template to the template volume. - VMTemplateStoragePoolVO templatePoolRef = _tmpltPoolDao.findByPoolTemplate(destPrimaryDataStore.getId(), templateOnPrimary.getId()); + VMTemplateStoragePoolVO templatePoolRef = _tmpltPoolDao.findByPoolTemplate(destPrimaryDataStore.getId(), templateOnPrimary.getId(), null); if (templatePoolRef == null) { throw new CloudRuntimeException("Failed to find template " + srcTemplateInfo.getUniqueName() + " in storage pool " + destPrimaryDataStore.getId()); @@ -1196,8 +1203,8 @@ public class VolumeServiceImpl implements VolumeService { @Override public AsyncCallFuture createVolumeFromTemplateAsync(VolumeInfo volume, long dataStoreId, TemplateInfo template) { PrimaryDataStore pd = dataStoreMgr.getPrimaryDataStore(dataStoreId); - TemplateInfo templateOnPrimaryStore = pd.getTemplate(template.getId()); - AsyncCallFuture future = new AsyncCallFuture(); + TemplateInfo templateOnPrimaryStore = pd.getTemplate(template.getId(), volume.getDeployAsIsConfiguration()); + AsyncCallFuture future = new AsyncCallFuture<>(); if (templateOnPrimaryStore == null) { createBaseImageAsync(volume, pd, template, future); @@ -1234,7 +1241,7 @@ public class VolumeServiceImpl implements VolumeService { volumeOnStore.processEvent(Event.CreateOnlyRequested); _volumeDetailsDao.addDetail(volume.getId(), SNAPSHOT_ID, Long.toString(snapshot.getId()), false); - CreateVolumeFromBaseImageContext context = new CreateVolumeFromBaseImageContext(null, volume, store, volumeOnStore, future, snapshot); + CreateVolumeFromBaseImageContext context = new CreateVolumeFromBaseImageContext(null, volume, store, volumeOnStore, future, snapshot, null); AsyncCallbackDispatcher caller = AsyncCallbackDispatcher.create(this); caller.setCallback(caller.getTarget().createVolumeFromSnapshotCallback(null, null)).setContext(context); motionSrv.copyAsync(snapshot, volumeOnStore, caller); @@ -2125,4 +2132,4 @@ public class VolumeServiceImpl implements VolumeService { volDao.remove(vol.getId()); } } -} +} \ No newline at end of file diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java index 740b66844b5..d48a5d9b101 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java @@ -598,7 +598,7 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co private VMTemplateVO createVMTemplateRecord(String vmInternalName, long guestOsId, long accountId) { Long nextTemplateId = vmTemplateDao.getNextInSequence(Long.class, "id"); VMTemplateVO templateVO = new VMTemplateVO(nextTemplateId, "Imported-from-" + vmInternalName, Storage.ImageFormat.OVA, false, false, false, Storage.TemplateType.USER, null, - false, 64, accountId, null, "Template imported from VM " + vmInternalName, false, guestOsId, false, HypervisorType.VMware, null, null, false, false, false); + false, 64, accountId, null, "Template imported from VM " + vmInternalName, false, guestOsId, false, HypervisorType.VMware, null, null, false, false, false, false); return vmTemplateDao.persist(templateVO); } @@ -619,7 +619,7 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co VMTemplateStoragePoolVO templateRef = templateStoragePoolDao.findByPoolPath(poolId, templatePath); if (templateRef == null) { templateRef = new VMTemplateStoragePoolVO(poolId, templateId, null, 100, VMTemplateStorageResourceAssoc.Status.DOWNLOADED, templatePath, null, null, templatePath, - templateSize); + templateSize, null); templateRef.setState(ObjectInDataStoreStateMachine.State.Ready); templateStoragePoolDao.persist(templateRef); } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java index c9f8b0c337a..7ffc88af531 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java @@ -17,7 +17,7 @@ package com.cloud.hypervisor.guru; import com.cloud.agent.api.storage.OVFPropertyTO; -import com.cloud.agent.api.to.DataStoreTO; +import com.cloud.agent.api.to.DeployAsIsInfoTO; import com.cloud.agent.api.to.DiskTO; import com.cloud.agent.api.to.NicTO; import com.cloud.agent.api.to.VirtualMachineTO; @@ -35,16 +35,14 @@ import com.cloud.network.dao.NetworkDao; import com.cloud.network.dao.NetworkVO; import com.cloud.storage.GuestOSHypervisorVO; import com.cloud.storage.GuestOSVO; -import com.cloud.storage.TemplateOVFPropertyVO; +import com.cloud.storage.ImageStore; import com.cloud.storage.VMTemplateStoragePoolVO; -import com.cloud.storage.VMTemplateStorageResourceAssoc; import com.cloud.storage.Volume; import com.cloud.storage.dao.GuestOSDao; import com.cloud.storage.dao.GuestOSHypervisorDao; -import com.cloud.storage.dao.TemplateOVFPropertiesDao; +import com.cloud.storage.dao.VMTemplateDetailsDao; import com.cloud.storage.dao.VMTemplatePoolDao; import com.cloud.template.VirtualMachineTemplate; -import com.cloud.utils.Pair; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.DomainRouterVO; import com.cloud.vm.NicProfile; @@ -70,7 +68,7 @@ import java.util.Map; import java.util.stream.Collectors; class VmwareVmImplementer { - private static final Logger LOG = Logger.getLogger(VmwareVmImplementer.class); + private static final Logger LOGGER = Logger.getLogger(VmwareVmImplementer.class); @Inject DomainRouterDao domainRouterDao; @@ -89,10 +87,10 @@ class VmwareVmImplementer { @Inject PrimaryDataStoreDao storagePoolDao; @Inject - TemplateOVFPropertiesDao templateOVFPropertiesDao; - @Inject VMTemplatePoolDao templateStoragePoolDao; @Inject + VMTemplateDetailsDao templateDetailsDao; + @Inject VmwareManager vmwareMgr; private Boolean globalNestedVirtualisationEnabled; @@ -116,10 +114,11 @@ class VmwareVmImplementer { VirtualMachineTO implement(VirtualMachineProfile vm, VirtualMachineTO to, long clusterId) { to.setBootloader(VirtualMachineTemplate.BootloaderType.HVM); - + boolean deployAsIs = vm.getTemplate().isDeployAsIs(); + HostVO host = hostDao.findById(vm.getVirtualMachine().getHostId()); Map details = to.getDetails(); if (details == null) - details = new HashMap(); + details = new HashMap<>(); VirtualMachine.Type vmType = vm.getType(); boolean userVm = !(vmType.equals(VirtualMachine.Type.DomainRouter) || vmType.equals(VirtualMachine.Type.ConsoleProxy) || vmType.equals(VirtualMachine.Type.SecondaryStorageVm)); @@ -133,7 +132,7 @@ class VmwareVmImplementer { try { VirtualEthernetCardType.valueOf(nicDeviceType); } catch (Exception e) { - LOG.warn("Invalid NIC device type " + nicDeviceType + " is specified in VM details, switch to default E1000"); + LOGGER.warn("Invalid NIC device type " + nicDeviceType + " is specified in VM details, switch to default E1000"); details.put(VmDetailConstants.NIC_ADAPTER, VirtualEthernetCardType.E1000.toString()); } } @@ -145,7 +144,7 @@ class VmwareVmImplementer { try { VirtualEthernetCardType.valueOf(nicDeviceType); } catch (Exception e) { - LOG.warn("Invalid NIC device type " + nicDeviceType + " is specified in VM details, switch to default E1000"); + LOGGER.warn("Invalid NIC device type " + nicDeviceType + " is specified in VM details, switch to default E1000"); details.put(VmDetailConstants.NIC_ADAPTER, VirtualEthernetCardType.E1000.toString()); } } @@ -172,7 +171,7 @@ class VmwareVmImplementer { GuestOSVO guestOS = guestOsDao.findByIdIncludingRemoved(vm.getVirtualMachine().getGuestOSId()); to.setOs(guestOS.getDisplayName()); to.setHostName(vm.getHostName()); - HostVO host = hostDao.findById(vm.getVirtualMachine().getHostId()); + GuestOSHypervisorVO guestOsMapping = null; if (host != null) { guestOsMapping = guestOsHypervisorDao.findByOsIdAndHypervisor(guestOS.getId(), Hypervisor.HypervisorType.VMware.toString(), host.getHypervisorVersion()); @@ -183,22 +182,55 @@ class VmwareVmImplementer { to.setPlatformEmulator(guestOsMapping.getGuestOsName()); } - List ovfProperties = getOvfPropertyList(vm, details); - - handleOvfProperties(vm, to, details, ovfProperties); + if (deployAsIs) { + List ovfProperties = getOvfPropertyList(vm, details); + handleOvfProperties(vm, to, details, ovfProperties); + setDeployAsIsParams(vm, to, details); + } setDetails(to, details); return to; } - private void setDetails(VirtualMachineTO to, Map details) { - if (LOG.isTraceEnabled()) { - for (String key: details.keySet()) { - LOG.trace(String.format("Detail for VM %s: %s => %s",to.getName(), key, details.get(key))); - } + private void setDeployAsIsParams(VirtualMachineProfile vm, VirtualMachineTO to, Map details) { + DeployAsIsInfoTO info = new DeployAsIsInfoTO(); + + String configuration = null; + if (details.containsKey(VmDetailConstants.DEPLOY_AS_IS_CONFIGURATION)) { + configuration = details.get(VmDetailConstants.DEPLOY_AS_IS_CONFIGURATION); + info.setDeploymentConfiguration(configuration); } - to.setDetails(details); + + // Deploy as-is disks are all allocated to the same storage pool + String deployAsIsStoreUuid = vm.getDisks().get(0).getData().getDataStore().getUuid(); + StoragePoolVO storagePoolVO = storagePoolDao.findByUuid(deployAsIsStoreUuid); + VMTemplateStoragePoolVO tmplRef = templateStoragePoolDao.findByPoolTemplate(storagePoolVO.getId(), vm.getTemplate().getId(), configuration); + if (tmplRef != null) { + info.setTemplatePath(tmplRef.getInstallPath()); + } + + info.setDeployAsIs(true); + to.setDeployAsIsInfo(info); + } + + private void setDetails(VirtualMachineTO to, Map details) { + Map detailsToSend = new HashMap<>(); + for (String key: details.keySet()) { + if (key.startsWith(ImageStore.OVF_EULA_SECTION_PREFIX) || + key.startsWith(ImageStore.OVF_HARDWARE_CONFIGURATION_PREFIX) || + key.startsWith(ImageStore.OVF_HARDWARE_ITEM_PREFIX)) { + if (LOGGER.isTraceEnabled()) { + LOGGER.trace(String.format("Discarding detail for VM %s: %s => %s", to.getName(), key, details.get(key))); + } + continue; + } + if (LOGGER.isTraceEnabled()) { + LOGGER.trace(String.format("Detail for VM %s: %s => %s", to.getName(), key, details.get(key))); + } + detailsToSend.put(key, details.get(key)); + } + to.setDetails(detailsToSend); } private void configureDomainRouterNicsAndDetails(VirtualMachineProfile vm, VirtualMachineTO to, Map details, List nicProfiles) { @@ -289,47 +321,40 @@ class VmwareVmImplementer { private void handleOvfProperties(VirtualMachineProfile vm, VirtualMachineTO to, Map details, List ovfProperties) { if (CollectionUtils.isNotEmpty(ovfProperties)) { removeOvfPropertiesFromDetails(ovfProperties, details); - String templateInstallPath = null; - List rootDiskList = vm.getDisks().stream().filter(x -> x.getType() == Volume.Type.ROOT).collect(Collectors.toList()); - if (rootDiskList.size() != 1) { + to.setOvfProperties(ovfProperties); + } + } + + private DiskTO getRootDiskTOFromVM(VirtualMachineProfile vm) { + DiskTO rootDiskTO; + List rootDiskList; + rootDiskList = vm.getDisks().stream().filter(x -> x.getType() == Volume.Type.ROOT).collect(Collectors.toList()); + if (rootDiskList.size() != 1) { + if (vm.getTemplate().isDeployAsIs()) { + rootDiskList = vm.getDisks().stream().filter(x -> x.getType() == null).collect(Collectors.toList()); + if (rootDiskList.size() < 1) { + throw new CloudRuntimeException("Did not find a template to serve as root disk for VM " + vm.getHostName()); + } + } else { throw new CloudRuntimeException("Did not find only one root disk for VM " + vm.getHostName()); } - - DiskTO rootDiskTO = rootDiskList.get(0); - DataStoreTO dataStore = rootDiskTO.getData().getDataStore(); - StoragePoolVO storagePoolVO = storagePoolDao.findByUuid(dataStore.getUuid()); - long dataCenterId = storagePoolVO.getDataCenterId(); - List pools = storagePoolDao.listByDataCenterId(dataCenterId); - for (StoragePoolVO pool : pools) { - VMTemplateStoragePoolVO ref = templateStoragePoolDao.findByPoolTemplate(pool.getId(), vm.getTemplateId()); - if (ref != null && ref.getDownloadState() == VMTemplateStorageResourceAssoc.Status.DOWNLOADED) { - templateInstallPath = ref.getInstallPath(); - break; - } - } - - if (templateInstallPath == null) { - throw new CloudRuntimeException("Did not find the template install path for template " + vm.getTemplateId() + " on zone " + dataCenterId); - } - - Pair> pair = new Pair>(templateInstallPath, ovfProperties); - to.setOvfProperties(pair); } + rootDiskTO = rootDiskList.get(0); + return rootDiskTO; } private List getOvfPropertyList(VirtualMachineProfile vm, Map details) { List ovfProperties = new ArrayList(); for (String detailKey : details.keySet()) { - if (detailKey.startsWith(ApiConstants.OVF_PROPERTIES)) { - String ovfPropKey = detailKey.replace(ApiConstants.OVF_PROPERTIES + "-", ""); - TemplateOVFPropertyVO templateOVFPropertyVO = templateOVFPropertiesDao.findByTemplateAndKey(vm.getTemplateId(), ovfPropKey); - if (templateOVFPropertyVO == null) { - LOG.warn(String.format("OVF property %s not found on template, discarding", ovfPropKey)); + if (detailKey.startsWith(ImageStore.ACS_PROPERTY_PREFIX)) { + OVFPropertyTO propertyTO = templateDetailsDao.findPropertyByTemplateAndKey(vm.getTemplateId(), detailKey); + String vmPropertyKey = detailKey.replace(ImageStore.ACS_PROPERTY_PREFIX, ""); + if (propertyTO == null) { + LOGGER.warn(String.format("OVF property %s not found on template, discarding", vmPropertyKey)); continue; } - String ovfValue = details.get(detailKey); - boolean isPassword = templateOVFPropertyVO.isPassword(); - OVFPropertyTO propertyTO = new OVFPropertyTO(ovfPropKey, ovfValue, isPassword); + propertyTO.setKey(vmPropertyKey); + propertyTO.setValue(details.get(detailKey)); ovfProperties.add(propertyTO); } } @@ -389,7 +414,7 @@ class VmwareVmImplementer { private void removeOvfPropertiesFromDetails(List ovfProperties, Map details) { for (OVFPropertyTO propertyTO : ovfProperties) { String key = propertyTO.getKey(); - details.remove(ApiConstants.OVF_PROPERTIES + "-" + key); + details.remove(ApiConstants.PROPERTIES + "-" + key); } } @@ -405,8 +430,8 @@ class VmwareVmImplementer { Boolean globalNestedVPerVMEnabled = getGlobalNestedVPerVMEnabled(); Boolean shouldEnableNestedVirtualization = shouldEnableNestedVirtualization(globalNestedVirtualisationEnabled, globalNestedVPerVMEnabled, localNestedV); - if(LOG.isDebugEnabled()) { - LOG.debug(String.format( + if(LOGGER.isDebugEnabled()) { + LOGGER.debug(String.format( "Due to '%B'(globalNestedVirtualisationEnabled) and '%B'(globalNestedVPerVMEnabled) I'm adding a flag with value %B to the vm configuration for Nested Virtualisation.", globalNestedVirtualisationEnabled, globalNestedVPerVMEnabled, diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareStorageManagerImpl.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareStorageManagerImpl.java index d8afb08f8df..2fcf1358d08 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareStorageManagerImpl.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareStorageManagerImpl.java @@ -591,7 +591,7 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { } String vmName = templateUuid; - hyperHost.importVmFromOVF(srcFileName, vmName, datastoreMo, "thin"); + hyperHost.importVmFromOVF(srcFileName, vmName, datastoreMo, "thin", null); VirtualMachineMO vmMo = hyperHost.findVmOnHyperHost(vmName); if (vmMo == null) { @@ -912,7 +912,7 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { VirtualMachineMO clonedVm = null; try { - hyperHost.importVmFromOVF(srcOVFFileName, newVolumeName, primaryDsMo, "thin"); + hyperHost.importVmFromOVF(srcOVFFileName, newVolumeName, primaryDsMo, "thin", null); clonedVm = hyperHost.findVmOnHyperHost(newVolumeName); if (clonedVm == null) { throw new Exception("Unable to create container VM for volume creation"); diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 909f5f76a78..5d611f4be67 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -16,10 +16,53 @@ // under the License. package com.cloud.hypervisor.vmware.resource; +import java.io.File; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.ConnectException; +import java.net.InetSocketAddress; +import java.net.URI; +import java.net.URL; +import java.nio.channels.SocketChannel; +import java.rmi.RemoteException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.Date; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.TimeZone; +import java.util.UUID; +import java.util.stream.Collectors; + +import javax.naming.ConfigurationException; +import javax.xml.datatype.XMLGregorianCalendar; + +import com.cloud.agent.api.to.DataTO; +import com.cloud.agent.api.to.DeployAsIsInfoTO; +import com.cloud.storage.ImageStore; import com.cloud.agent.api.ValidateVcenterDetailsCommand; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.storage.configdrive.ConfigDrive; import org.apache.cloudstack.storage.to.TemplateObjectTO; +import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.apache.cloudstack.utils.volume.VirtualMachineDiskInfo; +import org.apache.cloudstack.vm.UnmanagedInstanceTO; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.ArrayUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.math.NumberUtils; +import org.apache.log4j.Logger; +import org.apache.log4j.NDC; +import org.joda.time.Duration; import com.cloud.agent.IAgentControl; import com.cloud.agent.api.Answer; @@ -1796,6 +1839,21 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa s_logger.error(msg); throw new Exception(msg); } + + DiskTO[] specDisks = vmSpec.getDisks(); + DeployAsIsInfoTO deployAsIsInfo = vmSpec.getDeployAsIsInfo(); + boolean installAsIs = deployAsIsInfo != null && deployAsIsInfo.isDeployAsIs(); + if (installAsIs && dcMo.findVm(vmInternalCSName) == null) { + if (s_logger.isTraceEnabled()) { + s_logger.trace("Deploying OVA from as is"); + } + String deployAsIsTemplate = deployAsIsInfo.getTemplatePath(); + String destDatastore = getDatastoreFromSpecDisks(specDisks); + String deploymentConfiguration = deployAsIsInfo.getDeploymentConfiguration(); + vmInVcenter = _storageProcessor.cloneVMFromTemplate(deployAsIsTemplate, vmInternalCSName, destDatastore); + mapSpecDisksToClonedDisks(vmInVcenter, vmInternalCSName, specDisks); + } + String guestOsId = translateGuestOsIdentifier(vmSpec.getArch(), vmSpec.getOs(), vmSpec.getPlatformEmulator()).value(); DiskTO[] disks = validateDisks(vmSpec.getDisks()); assert (disks.length > 0); @@ -1816,6 +1874,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } VirtualMachineDiskInfoBuilder diskInfoBuilder = null; + VirtualDevice[] nicDevices = null; VirtualMachineMO vmMo = hyperHost.findVmOnHyperHost(vmInternalCSName); DiskControllerType systemVmScsiControllerType = DiskControllerType.lsilogic; int firstScsiControllerBusNum = 0; @@ -1829,6 +1888,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa // retrieve disk information before we tear down diskInfoBuilder = vmMo.getDiskInfoBuilder(); hasSnapshot = vmMo.hasSnapshot(); + nicDevices = vmMo.getNicDevices(); if (!hasSnapshot) vmMo.tearDownDevices(new Class[]{VirtualDisk.class, VirtualEthernetCard.class}); else @@ -1925,7 +1985,20 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } } + // The number of disks changed must be 0 for install as is, as the VM is a clone + //int disksChanges = !installAsIs ? disks.length : 0; int totalChangeDevices = disks.length + nics.length; + int hackDeviceCount = 0; + if (diskInfoBuilder != null) { + hackDeviceCount += diskInfoBuilder.getDiskCount(); + } + hackDeviceCount += nicDevices == null ? 0 : nicDevices.length; + if (s_logger.isTraceEnabled()) { + s_logger.trace(String.format("current count(s) desired: %d/ found:%d. now adding device to device count for vApp config ISO", totalChangeDevices, hackDeviceCount)); + } + if (vmSpec.getOvfProperties() != null) { + totalChangeDevices++; + } DiskTO volIso = null; if (vmSpec.getType() != VirtualMachine.Type.User) { @@ -2309,15 +2382,11 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa configureVideoCard(vmMo, vmSpec, vmConfigSpec); // Set OVF properties (if available) - Pair> ovfPropsMap = vmSpec.getOvfProperties(); - VmConfigInfo templateVappConfig = null; - List ovfProperties = null; - if (ovfPropsMap != null) { - String vmTemplate = ovfPropsMap.first(); - s_logger.info("Find VM template " + vmTemplate); - VirtualMachineMO vmTemplateMO = dcMo.findVm(vmTemplate); - templateVappConfig = vmTemplateMO.getConfigInfo().getVAppConfig(); - ovfProperties = ovfPropsMap.second(); + List ovfProperties = vmSpec.getOvfProperties(); + VmConfigInfo templateVappConfig; + if (ovfProperties != null) { + VirtualMachineMO templateVMmo = dcMo.findVm(deployAsIsInfo.getTemplatePath()); + templateVappConfig = templateVMmo.getConfigInfo().getVAppConfig(); // Set OVF properties (if available) if (CollectionUtils.isNotEmpty(ovfProperties)) { s_logger.info("Copying OVF properties from template and setting them to the values the user provided"); @@ -2414,6 +2483,85 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } } + private String getDatastoreFromSpecDisks(DiskTO[] specDisks) { + if (specDisks.length == 0) { + return null; + } + + Map> psDisksMap = Arrays.asList(specDisks).stream() + .filter(x -> x.getType() != Volume.Type.ISO && x.getData() != null && x.getData().getDataStore() != null) + .collect(Collectors.groupingBy(x -> x.getData().getDataStore().getUuid())); + + String dataStore; + if (MapUtils.isEmpty(psDisksMap)) { + s_logger.error("Could not find a destination datastore for VM volumes"); + return null; + } else { + dataStore = psDisksMap.keySet().iterator().next(); + if (psDisksMap.keySet().size() > 1) { + s_logger.info("Found multiple destination datastores for VM volumes, selecting " + dataStore); + } + } + return dataStore; + } + + /** + * Modify the specDisks information to match the cloned VM's disks (from vmMo VM) + */ + private void mapSpecDisksToClonedDisks(VirtualMachineMO vmMo, String vmInternalCSName, DiskTO[] specDisks) { + try { + s_logger.debug("Mapping spec disks information to cloned VM disks for VM " + vmInternalCSName); + if (vmMo != null && ArrayUtils.isNotEmpty(specDisks)) { + List vmDisks = vmMo.getVirtualDisks(); + List sortedDisks = Arrays.asList(sortVolumesByDeviceId(specDisks)) + .stream() + .filter(x -> x.getType() == Volume.Type.ROOT) + .collect(Collectors.toList()); + if (sortedDisks.size() != vmDisks.size()) { + s_logger.error("Different number of root disks spec vs cloned deploy-as-is VM disks: " + sortedDisks.size() + " - " + vmDisks.size()); + return; + } + for (int i = 0; i < sortedDisks.size(); i++) { + DiskTO specDisk = sortedDisks.get(i); + VirtualDisk vmDisk = vmDisks.get(i); + DataTO dataVolume = specDisk.getData(); + if (dataVolume instanceof VolumeObjectTO) { + VolumeObjectTO volumeObjectTO = (VolumeObjectTO) dataVolume; + if (!volumeObjectTO.getSize().equals(vmDisk.getCapacityInBytes())) { + s_logger.info("Mapped disk size is not the same as the cloned VM disk size: " + + volumeObjectTO.getSize() + " - " + vmDisk.getCapacityInBytes()); + } + VirtualDeviceBackingInfo backingInfo = vmDisk.getBacking(); + if (backingInfo instanceof VirtualDiskFlatVer2BackingInfo) { + VirtualDiskFlatVer2BackingInfo backing = (VirtualDiskFlatVer2BackingInfo) backingInfo; + String fileName = backing.getFileName(); + if (StringUtils.isNotBlank(fileName)) { + String[] fileNameParts = fileName.split(" "); + String datastoreUuid = fileNameParts[0].replace("[", "").replace("]", ""); + String relativePath = fileNameParts[1].split("/")[1].replace(".vmdk", ""); + String vmSpecDatastoreUuid = volumeObjectTO.getDataStore().getUuid().replaceAll("-", ""); + if (!datastoreUuid.equals(vmSpecDatastoreUuid)) { + s_logger.info("Mapped disk datastore UUID is not the same as the cloned VM datastore UUID: " + + datastoreUuid + " - " + vmSpecDatastoreUuid); + } + volumeObjectTO.setPath(relativePath); + specDisk.setPath(relativePath); + } else { + s_logger.error("Empty backing filename for volume " + volumeObjectTO.getName()); + } + } else { + s_logger.error("Could not get volume backing info for volume " + volumeObjectTO.getName()); + } + } + } + } + } catch (Exception e) { + String msg = "Error mapping deploy-as-is VM disks from cloned VM " + vmInternalCSName; + s_logger.error(msg, e); + throw new CloudRuntimeException(e); + } + } + private void setBootOptions(VirtualMachineTO vmSpec, String bootMode, VirtualMachineConfigSpec vmConfigSpec) { VirtualMachineBootOptions bootOptions = null; if (StringUtils.isNotBlank(bootMode) && !bootMode.equalsIgnoreCase("bios")) { @@ -2457,12 +2605,22 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa private Map> getOVFMap(List props) { Map> map = new HashMap<>(); for (OVFPropertyTO prop : props) { - Pair pair = new Pair<>(prop.getValue(), prop.isPassword()); + String value = getPropertyValue(prop); + Pair pair = new Pair<>(value, prop.isPassword()); map.put(prop.getKey(), pair); } return map; } + private String getPropertyValue(OVFPropertyTO prop) { + String type = prop.getType(); + String value = prop.getValue(); + if ("boolean".equalsIgnoreCase(type)) { + value = Boolean.parseBoolean(value) ? "True" : "False"; + } + return value; + } + /** * Set the properties section from existing vApp configuration and values set on ovfProperties */ @@ -2843,7 +3001,10 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa private static void configCustomExtraOption(List extraOptions, VirtualMachineTO vmSpec) { // we no longer to validation anymore for (Map.Entry entry : vmSpec.getDetails().entrySet()) { - if (entry.getKey().equalsIgnoreCase(VmDetailConstants.BOOT_MODE)) { + if (entry.getKey().equalsIgnoreCase(VmDetailConstants.BOOT_MODE) || + entry.getKey().startsWith(ImageStore.REQUIRED_NETWORK_PREFIX) || + entry.getKey().startsWith(ImageStore.ACS_PROPERTY_PREFIX) || + entry.getKey().startsWith(ImageStore.DISK_DEFINITION_PREFIX)) { continue; } OptionValue newVal = new OptionValue(); diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java index 2c4d9a14381..1f1056a09aa 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -795,95 +795,100 @@ public class VmwareStorageProcessor implements StorageProcessor { DatastoreMO dsMo = new DatastoreMO(context, morDatastore); String vmdkName = volume.getName(); - String vmdkFileBaseName; - if (srcStore == null) { - // create a root volume for blank VM (created from ISO) - String dummyVmName = hostService.getWorkerName(context, cmd, 0); + String vmdkFileBaseName = null; + if (template.isDeployAsIs()) { + s_logger.info("Volume from deploy-as-is template, no need to create the volume at this point, VM will be cloned"); + } else { + if (srcStore == null) { + // create a root volume for blank VM (created from ISO) + String dummyVmName = hostService.getWorkerName(context, cmd, 0); - try { - vmMo = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, dummyVmName, null); - if (vmMo == null) { - throw new Exception("Unable to create a dummy VM for volume creation"); + try { + vmMo = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, dummyVmName, null); + if (vmMo == null) { + throw new Exception("Unable to create a dummy VM for volume creation"); + } + + vmdkFileBaseName = vmMo.getVmdkFileBaseNames().get(0); + // we only use the first file in the pair, linked or not will not matter + String vmdkFilePair[] = VmwareStorageLayoutHelper.getVmdkFilePairDatastorePath(dsMo, null, vmdkFileBaseName, VmwareStorageLayoutType.CLOUDSTACK_LEGACY, true); + String volumeDatastorePath = vmdkFilePair[0]; + synchronized (this) { + s_logger.info("Delete file if exists in datastore to clear the way for creating the volume. file: " + volumeDatastorePath); + VmwareStorageLayoutHelper.deleteVolumeVmdkFiles(dsMo, vmdkName, dcMo, searchExcludedFolders); + vmMo.createDisk(volumeDatastorePath, (long)(volume.getSize() / (1024L * 1024L)), morDatastore, -1); + vmMo.detachDisk(volumeDatastorePath, false); + } + } finally { + s_logger.info("Destroy dummy VM after volume creation"); + if (vmMo != null) { + s_logger.warn("Unable to destroy a null VM ManagedObjectReference"); + vmMo.detachAllDisks(); + vmMo.destroy(); + } } + } else { + String templatePath = template.getPath(); + VirtualMachineMO vmTemplate = VmwareHelper.pickOneVmOnRunningHost(dcMo.findVmByNameAndLabel(templatePath), true); + if (vmTemplate == null) { + s_logger.warn("Template host in vSphere is not in connected state, request template reload"); + return new CopyCmdAnswer("Template host in vSphere is not in connected state, request template reload"); + } + + ManagedObjectReference morPool = hyperHost.getHyperHostOwnerResourcePool(); + ManagedObjectReference morCluster = hyperHost.getHyperHostCluster(); + if (template.getSize() != null){ + _fullCloneFlag = volume.getSize() > template.getSize() ? true : _fullCloneFlag; + } + if (!_fullCloneFlag) { + createVMLinkedClone(vmTemplate, dcMo, vmdkName, morDatastore, morPool); + } else { + createVMFullClone(vmTemplate, dcMo, dsMo, vmdkName, morDatastore, morPool); + } + + vmMo = new ClusterMO(context, morCluster).findVmOnHyperHost(vmdkName); + assert (vmMo != null); vmdkFileBaseName = vmMo.getVmdkFileBaseNames().get(0); - // we only use the first file in the pair, linked or not will not matter - String vmdkFilePair[] = VmwareStorageLayoutHelper.getVmdkFilePairDatastorePath(dsMo, null, vmdkFileBaseName, VmwareStorageLayoutType.CLOUDSTACK_LEGACY, true); - String volumeDatastorePath = vmdkFilePair[0]; - synchronized (this) { - s_logger.info("Delete file if exists in datastore to clear the way for creating the volume. file: " + volumeDatastorePath); - VmwareStorageLayoutHelper.deleteVolumeVmdkFiles(dsMo, vmdkName, dcMo, searchExcludedFolders); - vmMo.createDisk(volumeDatastorePath, (long)(volume.getSize() / (1024L * 1024L)), morDatastore, -1); - vmMo.detachDisk(volumeDatastorePath, false); - } - } finally { - s_logger.info("Destroy dummy VM after volume creation"); - if (vmMo != null) { - s_logger.warn("Unable to destroy a null VM ManagedObjectReference"); - vmMo.detachAllDisks(); - vmMo.destroy(); - } - } - } else { - String templatePath = template.getPath(); - VirtualMachineMO vmTemplate = VmwareHelper.pickOneVmOnRunningHost(dcMo.findVmByNameAndLabel(templatePath), true); - if (vmTemplate == null) { - s_logger.warn("Template host in vSphere is not in connected state, request template reload"); - return new CopyCmdAnswer("Template host in vSphere is not in connected state, request template reload"); - } - - ManagedObjectReference morPool = hyperHost.getHyperHostOwnerResourcePool(); - ManagedObjectReference morCluster = hyperHost.getHyperHostCluster(); - if (template.getSize() != null){ - _fullCloneFlag = volume.getSize() > template.getSize() ? true : _fullCloneFlag; - } - if (!_fullCloneFlag) { - createVMLinkedClone(vmTemplate, dcMo, vmdkName, morDatastore, morPool); - } else { - createVMFullClone(vmTemplate, dcMo, dsMo, vmdkName, morDatastore, morPool); - } - - vmMo = new ClusterMO(context, morCluster).findVmOnHyperHost(vmdkName); - assert (vmMo != null); - - vmdkFileBaseName = vmMo.getVmdkFileBaseNames().get(0); - s_logger.info("Move volume out of volume-wrapper VM " + vmdkFileBaseName); - String[] vmwareLayoutFilePair = VmwareStorageLayoutHelper.getVmdkFilePairDatastorePath(dsMo, vmdkName, vmdkFileBaseName, VmwareStorageLayoutType.VMWARE, !_fullCloneFlag); - String[] legacyCloudStackLayoutFilePair = VmwareStorageLayoutHelper.getVmdkFilePairDatastorePath(dsMo, vmdkName, vmdkFileBaseName, VmwareStorageLayoutType.CLOUDSTACK_LEGACY, !_fullCloneFlag); + s_logger.info("Move volume out of volume-wrapper VM " + vmdkFileBaseName); + String[] vmwareLayoutFilePair = VmwareStorageLayoutHelper.getVmdkFilePairDatastorePath(dsMo, vmdkName, vmdkFileBaseName, VmwareStorageLayoutType.VMWARE, !_fullCloneFlag); + String[] legacyCloudStackLayoutFilePair = VmwareStorageLayoutHelper.getVmdkFilePairDatastorePath(dsMo, vmdkName, vmdkFileBaseName, VmwareStorageLayoutType.CLOUDSTACK_LEGACY, !_fullCloneFlag); for (int i=0; i template.getSize() || _fullCloneFlag; + } + if (!_fullCloneFlag) { + createVMLinkedClone(vmMo, dcMo, cloneName, morDatastore, morPool); + } else { + createVMFullClone(vmMo, dcMo, dsMo, cloneName, morDatastore, morPool); + } + } + private Pair copyVolumeFromSecStorage(VmwareHypervisorHost hyperHost, String srcVolumePath, DatastoreMO dsMo, String secStorageUrl, long wait, String nfsVersion) throws Exception { String volumeFolder; @@ -3478,7 +3495,7 @@ public class VmwareStorageProcessor implements StorageProcessor { VirtualMachineMO clonedVm = null; try { - hyperHost.importVmFromOVF(srcOVFFileName, newVolumeName, primaryDsMo, "thin"); + hyperHost.importVmFromOVF(srcOVFFileName, newVolumeName, primaryDsMo, "thin", null); clonedVm = hyperHost.findVmOnHyperHost(newVolumeName); if (clonedVm == null) { throw new Exception("Unable to create container VM for volume creation"); @@ -3640,4 +3657,40 @@ public class VmwareStorageProcessor implements StorageProcessor { public Answer copyVolumeFromPrimaryToPrimary(CopyCommand cmd) { return null; } + + /** + * Return the cloned VM from the template + */ + public VirtualMachineMO cloneVMFromTemplate(String templateName, String cloneName, String templatePrimaryStoreUuid) { + try { + VmwareContext context = hostService.getServiceContext(null); + VmwareHypervisorHost hyperHost = hostService.getHyperHost(context, null); + DatacenterMO dcMo = new DatacenterMO(context, hyperHost.getHyperHostDatacenter()); + VirtualMachineMO templateMo = dcMo.findVm(templateName); + if (templateMo == null) { + throw new CloudRuntimeException(String.format("Unable to find template %s in vSphere", templateName)); + } + ManagedObjectReference morDatastore = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, templatePrimaryStoreUuid); + DatastoreMO dsMo = new DatastoreMO(context, morDatastore); + ManagedObjectReference morPool = hyperHost.getHyperHostOwnerResourcePool(); + if (morDatastore == null) { + throw new CloudRuntimeException("Unable to find datastore in vSphere"); + } + s_logger.info("Cloning VM " + cloneName + " from template " + templatePrimaryStoreUuid); + if (!_fullCloneFlag) { + createVMLinkedClone(templateMo, dcMo, cloneName, morDatastore, morPool); + } else { + createVMFullClone(templateMo, dcMo, dsMo, cloneName, morDatastore, morPool); + } + VirtualMachineMO vm = dcMo.findVm(cloneName); + if (vm == null) { + throw new CloudRuntimeException("Unable to get the cloned VM " + cloneName); + } + return vm; + } catch (Throwable e) { + String msg = "Error cloning VM from template in primary storage: %s" + e.getMessage(); + s_logger.error(msg, e); + throw new CloudRuntimeException(msg, e); + } + } } diff --git a/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/driver/DateraPrimaryDataStoreDriver.java b/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/driver/DateraPrimaryDataStoreDriver.java index 497960d1c23..fa1f3d4a646 100644 --- a/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/driver/DateraPrimaryDataStoreDriver.java +++ b/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/driver/DateraPrimaryDataStoreDriver.java @@ -955,7 +955,7 @@ public class DateraPrimaryDataStoreDriver implements PrimaryDataStoreDriver { } else if (dataType == DataObjectType.TEMPLATE) { s_logger.debug("Clone volume from a template"); - VMTemplateStoragePoolVO templatePoolRef = tmpltPoolDao.findByPoolTemplate(storagePoolId, dataObjectId); + VMTemplateStoragePoolVO templatePoolRef = tmpltPoolDao.findByPoolTemplate(storagePoolId, dataObjectId, null); if (templatePoolRef != null) { baseAppInstanceName = templatePoolRef.getLocalDownloadPath(); @@ -1114,7 +1114,7 @@ public class DateraPrimaryDataStoreDriver implements PrimaryDataStoreDriver { iqn = appInstance.getIqn(); VMTemplateStoragePoolVO templatePoolRef = tmpltPoolDao.findByPoolTemplate(storagePoolId, - templateInfo.getId()); + templateInfo.getId(), null); templatePoolRef.setInstallPath(DateraUtil.generateIqnPath(iqn)); templatePoolRef.setLocalDownloadPath(appInstance.getName()); @@ -1505,7 +1505,7 @@ public class DateraPrimaryDataStoreDriver implements PrimaryDataStoreDriver { DateraUtil.deleteAppInstance(conn, appInstanceName); VMTemplateStoragePoolVO templatePoolRef = tmpltPoolDao.findByPoolTemplate(storagePoolId, - templateInfo.getId()); + templateInfo.getId(), null); tmpltPoolDao.remove(templatePoolRef.getId()); diff --git a/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/driver/SolidFirePrimaryDataStoreDriver.java b/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/driver/SolidFirePrimaryDataStoreDriver.java index 22e4e952b3e..0651f2ea856 100644 --- a/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/driver/SolidFirePrimaryDataStoreDriver.java +++ b/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/driver/SolidFirePrimaryDataStoreDriver.java @@ -709,7 +709,7 @@ public class SolidFirePrimaryDataStoreDriver implements PrimaryDataStoreDriver { sfVolumeId = Long.parseLong(snapshotDetails.getValue()); } else if (dataObjectType == DataObjectType.TEMPLATE) { // get the cached template on this storage - VMTemplateStoragePoolVO templatePoolRef = vmTemplatePoolDao.findByPoolTemplate(storagePoolId, dataObjectId); + VMTemplateStoragePoolVO templatePoolRef = vmTemplatePoolDao.findByPoolTemplate(storagePoolId, dataObjectId, null); if (templatePoolRef != null) { sfVolumeId = Long.parseLong(templatePoolRef.getLocalDownloadPath()); @@ -1135,7 +1135,7 @@ public class SolidFirePrimaryDataStoreDriver implements PrimaryDataStoreDriver { String iqn = sfVolume.getIqn(); - VMTemplateStoragePoolVO templatePoolRef = vmTemplatePoolDao.findByPoolTemplate(storagePoolId, templateInfo.getId()); + VMTemplateStoragePoolVO templatePoolRef = vmTemplatePoolDao.findByPoolTemplate(storagePoolId, templateInfo.getId(), null); templatePoolRef.setInstallPath(iqn); templatePoolRef.setLocalDownloadPath(Long.toString(sfVolume.getId())); diff --git a/server/src/main/java/com/cloud/api/ApiDBUtils.java b/server/src/main/java/com/cloud/api/ApiDBUtils.java index e6b30775795..a2433ab634d 100644 --- a/server/src/main/java/com/cloud/api/ApiDBUtils.java +++ b/server/src/main/java/com/cloud/api/ApiDBUtils.java @@ -2004,16 +2004,16 @@ public class ApiDBUtils { return s_templateJoinDao.newUpdateResponse(vr); } - public static TemplateResponse newTemplateResponse(ResponseView view, TemplateJoinVO vr) { - return s_templateJoinDao.newTemplateResponse(view, vr); + public static TemplateResponse newTemplateResponse(EnumSet detailsView, ResponseView view, TemplateJoinVO vr) { + return s_templateJoinDao.newTemplateResponse(detailsView, view, vr); } public static TemplateResponse newIsoResponse(TemplateJoinVO vr) { return s_templateJoinDao.newIsoResponse(vr); } - public static TemplateResponse fillTemplateDetails(ResponseView view, TemplateResponse vrData, TemplateJoinVO vr) { - return s_templateJoinDao.setTemplateResponse(view, vrData, vr); + public static TemplateResponse fillTemplateDetails(EnumSet detailsView, ResponseView view, TemplateResponse vrData, TemplateJoinVO vr) { + return s_templateJoinDao.setTemplateResponse(detailsView, view, vrData, vr); } public static List newTemplateView(VirtualMachineTemplate vr) { diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index 2957a59b07e..51a5436fba5 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -1564,7 +1564,7 @@ public class ApiResponseHelper implements ResponseGenerator { tvo = ApiDBUtils.newTemplateView(result, zoneId, readyOnly); } - return ViewResponseHelper.createTemplateResponse(view, tvo.toArray(new TemplateJoinVO[tvo.size()])); + return ViewResponseHelper.createTemplateResponse(EnumSet.of(DomainDetails.all), view, tvo.toArray(new TemplateJoinVO[tvo.size()])); } @Override @@ -1581,7 +1581,7 @@ public class ApiResponseHelper implements ResponseGenerator { tvo.addAll(ApiDBUtils.newTemplateView(result, zoneId, readyOnly)); } } - return ViewResponseHelper.createTemplateResponse(view, tvo.toArray(new TemplateJoinVO[tvo.size()])); + return ViewResponseHelper.createTemplateResponse(EnumSet.of(DomainDetails.all), view, tvo.toArray(new TemplateJoinVO[tvo.size()])); } @Override diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java index 005968949c7..e8161bc4c65 100644 --- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java @@ -31,6 +31,7 @@ import java.util.stream.Stream; import javax.inject.Inject; +import com.cloud.storage.dao.VMTemplateDetailsDao; import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.affinity.AffinityGroupDomainMapVO; import org.apache.cloudstack.affinity.AffinityGroupResponse; @@ -71,7 +72,6 @@ import org.apache.cloudstack.api.command.user.project.ListProjectsCmd; import org.apache.cloudstack.api.command.user.resource.ListDetailOptionsCmd; import org.apache.cloudstack.api.command.user.securitygroup.ListSecurityGroupsCmd; import org.apache.cloudstack.api.command.user.tag.ListTagsCmd; -import org.apache.cloudstack.api.command.user.template.ListTemplateOVFProperties; import org.apache.cloudstack.api.command.user.template.ListTemplatesCmd; import org.apache.cloudstack.api.command.user.vm.ListVMsCmd; import org.apache.cloudstack.api.command.user.vmgroup.ListVMGroupsCmd; @@ -101,7 +101,6 @@ import org.apache.cloudstack.api.response.SecurityGroupResponse; import org.apache.cloudstack.api.response.ServiceOfferingResponse; import org.apache.cloudstack.api.response.StoragePoolResponse; import org.apache.cloudstack.api.response.StorageTagResponse; -import org.apache.cloudstack.api.response.TemplateOVFPropertyResponse; import org.apache.cloudstack.api.response.TemplateResponse; import org.apache.cloudstack.api.response.UserResponse; import org.apache.cloudstack.api.response.UserVmResponse; @@ -125,7 +124,6 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; -import com.cloud.agent.api.storage.OVFProperty; import com.cloud.api.query.dao.AccountJoinDao; import com.cloud.api.query.dao.AffinityGroupJoinDao; import com.cloud.api.query.dao.AsyncJobJoinDao; @@ -214,11 +212,9 @@ import com.cloud.storage.Storage; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.Storage.TemplateType; import com.cloud.storage.StoragePoolTagVO; -import com.cloud.storage.TemplateOVFPropertyVO; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.Volume; import com.cloud.storage.dao.StoragePoolTagsDao; -import com.cloud.storage.dao.TemplateOVFPropertiesDao; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.tags.ResourceTagVO; import com.cloud.tags.dao.ResourceTagDao; @@ -405,7 +401,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q ManagementServerHostDao managementServerHostDao; @Inject - TemplateOVFPropertiesDao templateOVFPropertiesDao; + VMTemplateDetailsDao vmTemplateDetailsDao; @Inject public VpcVirtualNetworkApplianceService routerService; @@ -2919,6 +2915,9 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q ServiceOfferingVO currentVmOffering = null; Boolean isRecursive = cmd.isRecursive(); Long zoneId = cmd.getZoneId(); + Integer cpuNumber = cmd.getCpuNumber(); + Integer memory = cmd.getMemory(); + Integer cpuSpeed = cmd.getCpuSpeed(); SearchCriteria sc = _srvOfferingJoinDao.createSearchCriteria(); if (!_accountMgr.isRootAdmin(caller.getId()) && isSystem) { @@ -2970,35 +2969,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q if (caller.getType() == Account.ACCOUNT_TYPE_NORMAL) { throw new InvalidParameterValueException("Only ROOT admins and Domain admins can list service offerings with isrecursive=true"); } - }/* else { // domain + all ancestors - // find all domain Id up to root domain for this account - List domainIds = new ArrayList(); - DomainVO domainRecord; - if (vmId != null) { - UserVmVO vmInstance = _userVmDao.findById(vmId); - domainRecord = _domainDao.findById(vmInstance.getDomainId()); - if (domainRecord == null) { - s_logger.error("Could not find the domainId for vmId:" + vmId); - throw new CloudAuthenticationException("Could not find the domainId for vmId:" + vmId); - } - } else { - domainRecord = _domainDao.findById(caller.getDomainId()); - if (domainRecord == null) { - s_logger.error("Could not find the domainId for account:" + caller.getAccountName()); - throw new CloudAuthenticationException("Could not find the domainId for account:" + caller.getAccountName()); - } - } - domainIds.add(domainRecord.getId()); - while (domainRecord.getParent() != null) { - domainRecord = _domainDao.findById(domainRecord.getParent()); - domainIds.add(domainRecord.getId()); - } - - SearchCriteria spc = _srvOfferingJoinDao.createSearchCriteria(); - spc.addOr("domainId", SearchCriteria.Op.IN, domainIds.toArray()); - spc.addOr("domainId", SearchCriteria.Op.NULL); // include public offering as well - sc.addAnd("domainId", SearchCriteria.Op.SC, spc); - }*/ + } } else { // for root users if (caller.getDomainId() != 1 && isSystem) { // NON ROOT admin @@ -3045,6 +3016,37 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q sc.addAnd("zoneId", SearchCriteria.Op.SC, zoneSC); } + if (cpuNumber != null) { + SearchCriteria cpuConstraintSearchCriteria = _srvOfferingJoinDao.createSearchCriteria(); + cpuConstraintSearchCriteria.addAnd("minCpu", Op.LTEQ, cpuNumber); + cpuConstraintSearchCriteria.addAnd("maxCpu", Op.GTEQ, cpuNumber); + + SearchCriteria cpuSearchCriteria = _srvOfferingJoinDao.createSearchCriteria(); + cpuSearchCriteria.addOr("minCpu", Op.NULL); + cpuSearchCriteria.addOr("constraints", Op.SC, cpuConstraintSearchCriteria); + + sc.addAnd("cpuConstraints", SearchCriteria.Op.SC, cpuSearchCriteria); + } + + if (memory != null) { + SearchCriteria memoryConstraintSearchCriteria = _srvOfferingJoinDao.createSearchCriteria(); + memoryConstraintSearchCriteria.addAnd("minMemory", Op.LTEQ, memory); + memoryConstraintSearchCriteria.addAnd("maxMemory", Op.GTEQ, memory); + + SearchCriteria memSearchCriteria = _srvOfferingJoinDao.createSearchCriteria(); + memSearchCriteria.addOr("minMemory", Op.NULL); + memSearchCriteria.addOr("memconstraints", Op.SC, memoryConstraintSearchCriteria); + + sc.addAnd("memoryConstraints", SearchCriteria.Op.SC, memSearchCriteria); + } + + if (cpuSpeed != null) { + SearchCriteria cpuSpeedSearchCriteria = _srvOfferingJoinDao.createSearchCriteria(); + cpuSpeedSearchCriteria.addOr("speed", Op.NULL); + cpuSpeedSearchCriteria.addOr("speed", Op.EQ, cpuSpeed); + sc.addAnd("cpuspeedconstraints", SearchCriteria.Op.SC, cpuSpeedSearchCriteria); + } + Pair, Integer> result = _srvOfferingJoinDao.searchAndCount(sc, searchFilter); //Couldn't figure out a smart way to filter offerings based on tags in sql so doing it in Java. @@ -3325,7 +3327,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q respView = ResponseView.Full; } - List templateResponses = ViewResponseHelper.createTemplateResponse(respView, result.first().toArray(new TemplateJoinVO[result.first().size()])); + List templateResponses = ViewResponseHelper.createTemplateResponse(cmd.getDetails(), respView, result.first().toArray(new TemplateJoinVO[result.first().size()])); response.setResponses(templateResponses, result.second()); return response; } @@ -4095,29 +4097,6 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q return mgmtResponse; } - @Override - public ListResponse listTemplateOVFProperties(ListTemplateOVFProperties cmd) { - ListResponse response = new ListResponse<>(); - List result = new ArrayList<>(); - Long templateId = cmd.getTemplateId(); - List ovfProperties = templateOVFPropertiesDao.listByTemplateId(templateId); - for (OVFProperty property : ovfProperties) { - TemplateOVFPropertyResponse propertyResponse = new TemplateOVFPropertyResponse(); - propertyResponse.setKey(property.getKey()); - propertyResponse.setType(property.getType()); - propertyResponse.setValue(property.getValue()); - propertyResponse.setQualifiers(property.getQualifiers()); - propertyResponse.setUserConfigurable(property.isUserConfigurable()); - propertyResponse.setLabel(property.getLabel()); - propertyResponse.setDescription(property.getDescription()); - propertyResponse.setPassword(property.isPassword()); - propertyResponse.setObjectName("ovfproperty"); - result.add(propertyResponse); - } - response.setResponses(result); - return response; - } - @Override public List listRouterHealthChecks(GetRouterHealthCheckResultsCmd cmd) { s_logger.info("Executing health check command " + cmd); diff --git a/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java b/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java index 88a7639ac91..d91fa7bf480 100644 --- a/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java +++ b/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java @@ -26,6 +26,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.affinity.AffinityGroupResponse; import org.apache.cloudstack.api.ApiConstants.DomainDetails; import org.apache.cloudstack.api.ApiConstants.HostDetails; @@ -583,17 +584,17 @@ public class ViewResponseHelper { return respList; } - public static List createTemplateResponse(ResponseView view, TemplateJoinVO... templates) { + public static List createTemplateResponse(EnumSet detailsView, ResponseView view, TemplateJoinVO... templates) { LinkedHashMap vrDataList = new LinkedHashMap(); for (TemplateJoinVO vr : templates) { TemplateResponse vrData = vrDataList.get(vr.getTempZonePair()); if (vrData == null) { // first time encountering this volume - vrData = ApiDBUtils.newTemplateResponse(view, vr); + vrData = ApiDBUtils.newTemplateResponse(detailsView, view, vr); } else{ // update tags - vrData = ApiDBUtils.fillTemplateDetails(view, vrData, vr); + vrData = ApiDBUtils.fillTemplateDetails(detailsView, view, vrData, vr); } vrDataList.put(vr.getTempZonePair(), vrData); } @@ -609,7 +610,7 @@ public class ViewResponseHelper { vrData = ApiDBUtils.newTemplateUpdateResponse(vr); } else { // update tags - vrData = ApiDBUtils.fillTemplateDetails(view, vrData, vr); + vrData = ApiDBUtils.fillTemplateDetails(EnumSet.of(DomainDetails.all), view, vrData, vr); } vrDataList.put(vr.getId(), vrData); } @@ -625,7 +626,7 @@ public class ViewResponseHelper { vrData = ApiDBUtils.newIsoResponse(vr); } else { // update tags - vrData = ApiDBUtils.fillTemplateDetails(view, vrData, vr); + vrData = ApiDBUtils.fillTemplateDetails(EnumSet.of(DomainDetails.all), view, vrData, vr); } vrDataList.put(vr.getTempZonePair(), vrData); } diff --git a/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDao.java b/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDao.java index c9d7eba48b2..58cb886594f 100644 --- a/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDao.java +++ b/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDao.java @@ -16,8 +16,10 @@ // under the License. package com.cloud.api.query.dao; +import java.util.EnumSet; import java.util.List; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ResponseObject.ResponseView; import org.apache.cloudstack.api.response.TemplateResponse; @@ -30,13 +32,13 @@ import com.cloud.utils.db.SearchCriteria; public interface TemplateJoinDao extends GenericDao { - TemplateResponse newTemplateResponse(ResponseView view, TemplateJoinVO tmpl); + TemplateResponse newTemplateResponse(EnumSet detailsView, ResponseView view, TemplateJoinVO tmpl); TemplateResponse newIsoResponse(TemplateJoinVO tmpl); TemplateResponse newUpdateResponse(TemplateJoinVO tmpl); - TemplateResponse setTemplateResponse(ResponseView view, TemplateResponse tmplData, TemplateJoinVO tmpl); + TemplateResponse setTemplateResponse(EnumSet detailsView, ResponseView view, TemplateResponse tmplData, TemplateJoinVO tmpl); List newTemplateView(VirtualMachineTemplate tmpl); diff --git a/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDaoImpl.java index 6b700a255ae..e51dd1e3f9c 100644 --- a/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDaoImpl.java @@ -17,6 +17,7 @@ package com.cloud.api.query.dao; import java.util.ArrayList; +import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -25,6 +26,11 @@ import java.util.Set; import javax.inject.Inject; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.utils.security.DigestHelper; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; + import org.apache.cloudstack.api.ResponseObject.ResponseView; import org.apache.cloudstack.api.response.ChildTemplateResponse; import org.apache.cloudstack.api.response.TemplateResponse; @@ -147,7 +153,7 @@ public class TemplateJoinDaoImpl extends GenericDaoBaseWithTagInformation detailsView, ResponseView view, TemplateJoinVO template) { List templatesInStore = _templateStoreDao.listByTemplateNotBypassed(template.getId()); List> downloadProgressDetails = new ArrayList(); HashMap downloadDetailInImageStores = null; @@ -158,6 +164,7 @@ public class TemplateJoinDaoImpl extends GenericDaoBaseWithTagInformation details = _templateDetailsDao.listDetailsKeyPairs(template.getId()); - templateResponse.setDetails(details); + if (detailsView.contains(ApiConstants.DomainDetails.all)) { + Map details = _templateDetailsDao.listDetailsKeyPairs(template.getId()); + templateResponse.setDetails(details); + } // update tag information long tag_id = template.getTagId(); @@ -240,6 +249,7 @@ public class TemplateJoinDaoImpl extends GenericDaoBaseWithTagInformation details = templateResponse.getDetails(); - if (details == null) { - details = new HashMap<>(); + public TemplateResponse setTemplateResponse(EnumSet detailsView, ResponseView view, TemplateResponse templateResponse, TemplateJoinVO template) { + if (detailsView.contains(ApiConstants.DomainDetails.all)) { + // update details map + String key = template.getDetailName(); + if (key != null) { + templateResponse.addDetail(key, template.getDetailValue()); } - details.put(template.getDetailName(), template.getDetailValue()); - templateResponse.setDetails(details); } // update tag information @@ -496,7 +503,6 @@ public class TemplateJoinDaoImpl extends GenericDaoBaseWithTagInformation resourceDetails = new HashMap(); for (UserVmDetailVO userVmDetailVO : vmDetails) { - if (!userVmDetailVO.getName().startsWith(ApiConstants.OVF_PROPERTIES) || - (UserVmManager.DisplayVMOVFProperties.value() && userVmDetailVO.getName().startsWith(ApiConstants.OVF_PROPERTIES))) { + if (!userVmDetailVO.getName().startsWith(ApiConstants.PROPERTIES) || + (UserVmManager.DisplayVMOVFProperties.value() && userVmDetailVO.getName().startsWith(ApiConstants.PROPERTIES))) { resourceDetails.put(userVmDetailVO.getName(), userVmDetailVO.getValue()); } if ((ApiConstants.BootType.UEFI.toString()).equalsIgnoreCase(userVmDetailVO.getName())) { diff --git a/server/src/main/java/com/cloud/api/query/vo/ServiceOfferingJoinVO.java b/server/src/main/java/com/cloud/api/query/vo/ServiceOfferingJoinVO.java index 4f8932ad4cf..a3e1cb14723 100644 --- a/server/src/main/java/com/cloud/api/query/vo/ServiceOfferingJoinVO.java +++ b/server/src/main/java/com/cloud/api/query/vo/ServiceOfferingJoinVO.java @@ -175,6 +175,18 @@ public class ServiceOfferingJoinVO extends BaseViewVO implements InternalIdentit @Column(name = "cache_mode") String cacheMode; + @Column(name = "min_cpu") + Integer minCpu; + + @Column(name = "max_cpu") + Integer maxCpu; + + @Column(name = "min_memory") + Integer minMemory; + + @Column(name = "max_memory") + Integer maxMemory; + public ServiceOfferingJoinVO() { } @@ -356,4 +368,20 @@ public class ServiceOfferingJoinVO extends BaseViewVO implements InternalIdentit public String getCacheMode() { return cacheMode; } + + public Integer getMinCpu() { + return minCpu; + } + + public Integer getMaxCpu() { + return maxCpu; + } + + public Integer getMinMemory() { + return minMemory; + } + + public Integer getMaxMemory() { + return maxMemory; + } } diff --git a/server/src/main/java/com/cloud/api/query/vo/TemplateJoinVO.java b/server/src/main/java/com/cloud/api/query/vo/TemplateJoinVO.java index 25e3b0b5ff5..91bb76336cc 100644 --- a/server/src/main/java/com/cloud/api/query/vo/TemplateJoinVO.java +++ b/server/src/main/java/com/cloud/api/query/vo/TemplateJoinVO.java @@ -231,6 +231,9 @@ public class TemplateJoinVO extends BaseViewWithTagInformationVO implements Cont @Column(name = "direct_download") private boolean directDownload; + @Column(name = "deploy_as_is") + private boolean deployAsIs; + public TemplateJoinVO() { } @@ -490,6 +493,10 @@ public class TemplateJoinVO extends BaseViewWithTagInformationVO implements Cont return directDownload; } + public boolean isDeployAsIs() { + return deployAsIs; + } + public Object getParentTemplateId() { return parentTemplateId; } diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index 83f3fc05d98..844f75997d5 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -457,7 +457,6 @@ import org.apache.cloudstack.api.command.user.template.CreateTemplateCmd; import org.apache.cloudstack.api.command.user.template.DeleteTemplateCmd; import org.apache.cloudstack.api.command.user.template.ExtractTemplateCmd; import org.apache.cloudstack.api.command.user.template.GetUploadParamsForTemplateCmd; -import org.apache.cloudstack.api.command.user.template.ListTemplateOVFProperties; import org.apache.cloudstack.api.command.user.template.ListTemplatePermissionsCmd; import org.apache.cloudstack.api.command.user.template.ListTemplatesCmd; import org.apache.cloudstack.api.command.user.template.RegisterTemplateCmd; @@ -3185,7 +3184,6 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe cmdList.add(RevokeTemplateDirectDownloadCertificateCmd.class); cmdList.add(ListMgmtsCmd.class); cmdList.add(GetUploadParamsForIsoCmd.class); - cmdList.add(ListTemplateOVFProperties.class); cmdList.add(GetRouterHealthCheckResultsCmd.class); cmdList.add(StartRollingMaintenanceCmd.class); cmdList.add(MigrateSecondaryStorageDataCmd.class); diff --git a/server/src/main/java/com/cloud/storage/ImageStoreUploadMonitorImpl.java b/server/src/main/java/com/cloud/storage/ImageStoreUploadMonitorImpl.java index a2f97e84127..1ce5b362eb9 100755 --- a/server/src/main/java/com/cloud/storage/ImageStoreUploadMonitorImpl.java +++ b/server/src/main/java/com/cloud/storage/ImageStoreUploadMonitorImpl.java @@ -408,11 +408,11 @@ public class ImageStoreUploadMonitorImpl extends ManagerBase implements ImageSto VMTemplateVO templateUpdate = _templateDao.createForUpdate(); templateUpdate.setSize(answer.getVirtualSize()); _templateDao.update(tmpTemplate.getId(), templateUpdate); - // For multi-disk OVA, check and create data disk templates + // For multi-disk OVA, check and create data disk templates or root disks as details if (tmpTemplate.getFormat().equals(Storage.ImageFormat.OVA)) { final DataStore store = dataStoreManager.getDataStore(templateDataStore.getDataStoreId(), templateDataStore.getDataStoreRole()); final TemplateInfo templateInfo = templateFactory.getTemplate(tmpTemplate.getId(), store); - if (!templateService.createOvaDataDiskTemplates(templateInfo)) { + if (!templateService.createOvaDataDiskTemplates(templateInfo, template.isDeployAsIs())) { tmpTemplateDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.ABANDONED); tmpTemplateDataStore.setState(State.Failed); stateMachine.transitTo(tmpTemplate, VirtualMachineTemplate.Event.OperationFailed, null, _templateDao); diff --git a/server/src/main/java/com/cloud/storage/TemplateProfile.java b/server/src/main/java/com/cloud/storage/TemplateProfile.java index 304b652a589..b90409480bc 100644 --- a/server/src/main/java/com/cloud/storage/TemplateProfile.java +++ b/server/src/main/java/com/cloud/storage/TemplateProfile.java @@ -52,6 +52,7 @@ public class TemplateProfile { Boolean isDynamicallyScalable; TemplateType templateType; Boolean directDownload; + Boolean deployAsIs; Long size; public TemplateProfile(Long templateId, Long userId, String name, String displayText, Integer bits, Boolean passwordEnabled, Boolean requiresHvm, String url, @@ -95,7 +96,7 @@ public class TemplateProfile { Boolean isPublic, Boolean featured, Boolean isExtractable, ImageFormat format, Long guestOsId, List zoneId, HypervisorType hypervisorType, String accountName, Long domainId, Long accountId, String chksum, Boolean bootable, String templateTag, Map details, - Boolean sshKeyEnabled, Long imageStoreId, Boolean isDynamicallyScalable, TemplateType templateType, Boolean directDownload) { + Boolean sshKeyEnabled, Long imageStoreId, Boolean isDynamicallyScalable, TemplateType templateType, Boolean directDownload, Boolean deployAsIs) { this(templateId, userId, name, @@ -122,6 +123,7 @@ public class TemplateProfile { this.isDynamicallyScalable = isDynamicallyScalable; this.templateType = templateType; this.directDownload = directDownload; + this.deployAsIs = deployAsIs; } public Long getTemplateId() { @@ -331,4 +333,8 @@ public class TemplateProfile { public void setSize(Long size) { this.size = size; } + + public boolean isDeployAsIs() { + return this.deployAsIs; + } } diff --git a/server/src/main/java/com/cloud/storage/upload/params/UploadParams.java b/server/src/main/java/com/cloud/storage/upload/params/UploadParams.java index 0d42b760b6d..be8319c9e57 100644 --- a/server/src/main/java/com/cloud/storage/upload/params/UploadParams.java +++ b/server/src/main/java/com/cloud/storage/upload/params/UploadParams.java @@ -46,4 +46,5 @@ public interface UploadParams { boolean isDynamicallyScalable(); boolean isRoutingType(); boolean isDirectDownload(); + boolean isDeployAsIs(); } diff --git a/server/src/main/java/com/cloud/storage/upload/params/UploadParamsBase.java b/server/src/main/java/com/cloud/storage/upload/params/UploadParamsBase.java index 67b04f7b480..e5bc1a3c906 100644 --- a/server/src/main/java/com/cloud/storage/upload/params/UploadParamsBase.java +++ b/server/src/main/java/com/cloud/storage/upload/params/UploadParamsBase.java @@ -214,6 +214,11 @@ public abstract class UploadParamsBase implements UploadParams { return false; } + @Override + public boolean isDeployAsIs() { + return false; + } + void setIso(boolean iso) { isIso = iso; } diff --git a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java index 80ca46912f2..95fe38b7189 100644 --- a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java +++ b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java @@ -25,6 +25,11 @@ import java.util.concurrent.ExecutionException; import javax.inject.Inject; +import com.cloud.configuration.Config; +import com.cloud.storage.dao.VMTemplateDetailsDao; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallback; +import com.cloud.utils.db.TransactionStatus; import org.apache.cloudstack.agent.directdownload.CheckUrlAnswer; import org.apache.cloudstack.agent.directdownload.CheckUrlCommand; import org.apache.cloudstack.api.command.user.iso.DeleteIsoCmd; @@ -60,7 +65,6 @@ import org.apache.log4j.Logger; import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; import com.cloud.alert.AlertManager; -import com.cloud.configuration.Config; import com.cloud.configuration.Resource.ResourceType; import com.cloud.dc.DataCenterVO; import com.cloud.dc.dao.DataCenterDao; @@ -89,9 +93,6 @@ import com.cloud.utils.Pair; import com.cloud.utils.UriUtils; import com.cloud.utils.db.DB; import com.cloud.utils.db.EntityManager; -import com.cloud.utils.db.Transaction; -import com.cloud.utils.db.TransactionCallback; -import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; public class HypervisorTemplateAdapter extends TemplateAdapterBase { @@ -126,6 +127,8 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { ResourceManager resourceManager; @Inject VMTemplateDao templateDao; + @Inject + private VMTemplateDetailsDao templateDetailsDao; @Override public String getName() { @@ -241,12 +244,14 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { private void createTemplateWithinZone(Long zId, TemplateProfile profile, VMTemplateVO template) { // find all eligible image stores for this zone scope - List imageStores = storeMgr.getImageStoresByScopeExcludingReadOnly(new ZoneScope(zId)); + List imageStores = storeMgr.getImageStoresByScope(new ZoneScope(zId)); if (imageStores == null || imageStores.size() == 0) { throw new CloudRuntimeException("Unable to find image store to download template " + profile.getTemplate()); } Set zoneSet = new HashSet(); + Collections.shuffle(imageStores); + // For private templates choose a random store. TODO - Have a better algorithm based on size, no. of objects, load etc. for (DataStore imageStore : imageStores) { // skip data stores for a disabled zone Long zoneId = imageStore.getScope().getScopeId(); @@ -306,7 +311,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { zoneId = profile.getZoneIdList().get(0); // find all eligible image stores for this zone scope - List imageStores = storeMgr.getImageStoresByScopeExcludingReadOnly(new ZoneScope(zoneId)); + List imageStores = storeMgr.getImageStoresByScope(new ZoneScope(zoneId)); if (imageStores == null || imageStores.size() == 0) { throw new CloudRuntimeException("Unable to find image store to download template " + profile.getTemplate()); } @@ -590,6 +595,9 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { Pair, Long> tmplt = new Pair, Long>(VirtualMachineTemplate.class, template.getId()); _messageBus.publish(_name, EntityManager.MESSAGE_REMOVE_ENTITY_EVENT, PublishScope.LOCAL, tmplt); + // Remove template details + templateDetailsDao.removeDetails(template.getId()); + } return success; diff --git a/server/src/main/java/com/cloud/template/TemplateAdapter.java b/server/src/main/java/com/cloud/template/TemplateAdapter.java index c048ceaf1fc..86dd0d3cad5 100644 --- a/server/src/main/java/com/cloud/template/TemplateAdapter.java +++ b/server/src/main/java/com/cloud/template/TemplateAdapter.java @@ -72,13 +72,12 @@ public interface TemplateAdapter extends Adapter { boolean delete(TemplateProfile profile); - TemplateProfile prepare(boolean isIso, Long userId, String name, String displayText, Integer bits, Boolean passwordEnabled, Boolean requiresHVM, String url, - Boolean isPublic, Boolean featured, Boolean isExtractable, String format, Long guestOSId, List zoneId, HypervisorType hypervisorType, String accountName, - Long domainId, String chksum, Boolean bootable, Map details, boolean directDownload) throws ResourceAllocationException; + TemplateProfile prepare(boolean isIso, Long userId, String name, String displayText, Integer bits, Boolean passwordEnabled, Boolean requiresHVM, String url, Boolean isPublic, + Boolean featured, Boolean isExtractable, String format, Long guestOSId, List zoneId, HypervisorType hypervisorType, String accountName, Long domainId, String chksum, Boolean bootable, Map details, boolean directDownload, + boolean deployAsIs) throws ResourceAllocationException; - TemplateProfile prepare(boolean isIso, long userId, String name, String displayText, Integer bits, Boolean passwordEnabled, Boolean requiresHVM, String url, - Boolean isPublic, Boolean featured, Boolean isExtractable, String format, Long guestOSId, List zoneId, HypervisorType hypervisorType, String chksum, - Boolean bootable, String templateTag, Account templateOwner, Map details, Boolean sshKeyEnabled, String imageStoreUuid, Boolean isDynamicallyScalable, - TemplateType templateType, boolean directDownload) throws ResourceAllocationException; + TemplateProfile prepare(boolean isIso, long userId, String name, String displayText, Integer bits, Boolean passwordEnabled, Boolean requiresHVM, String url, Boolean isPublic, + Boolean featured, Boolean isExtractable, String format, Long guestOSId, List zoneId, HypervisorType hypervisorType, String chksum, Boolean bootable, String templateTag, Account templateOwner, Map details, Boolean sshKeyEnabled, String imageStoreUuid, Boolean isDynamicallyScalable, + TemplateType templateType, boolean directDownload, boolean deployAsIs) throws ResourceAllocationException; } diff --git a/server/src/main/java/com/cloud/template/TemplateAdapterBase.java b/server/src/main/java/com/cloud/template/TemplateAdapterBase.java index 0e88c147f51..d016fed4a2e 100644 --- a/server/src/main/java/com/cloud/template/TemplateAdapterBase.java +++ b/server/src/main/java/com/cloud/template/TemplateAdapterBase.java @@ -129,16 +129,16 @@ public abstract class TemplateAdapterBase extends AdapterBase implements Templat @Override public TemplateProfile prepare(boolean isIso, Long userId, String name, String displayText, Integer bits, Boolean passwordEnabled, Boolean requiresHVM, String url, Boolean isPublic, Boolean featured, Boolean isExtractable, String format, Long guestOSId, List zoneId, HypervisorType hypervisorType, String accountName, - Long domainId, String chksum, Boolean bootable, Map details, boolean directDownload) throws ResourceAllocationException { + Long domainId, String chksum, Boolean bootable, Map details, boolean directDownload, boolean deployAsIs) throws ResourceAllocationException { return prepare(isIso, userId, name, displayText, bits, passwordEnabled, requiresHVM, url, isPublic, featured, isExtractable, format, guestOSId, zoneId, - hypervisorType, chksum, bootable, null, null, details, false, null, false, TemplateType.USER, directDownload); + hypervisorType, chksum, bootable, null, null, details, false, null, false, TemplateType.USER, directDownload, deployAsIs); } @Override public TemplateProfile prepare(boolean isIso, long userId, String name, String displayText, Integer bits, Boolean passwordEnabled, Boolean requiresHVM, String url, Boolean isPublic, Boolean featured, Boolean isExtractable, String format, Long guestOSId, List zoneIdList, HypervisorType hypervisorType, String chksum, Boolean bootable, String templateTag, Account templateOwner, Map details, Boolean sshkeyEnabled, String imageStoreUuid, Boolean isDynamicallyScalable, - TemplateType templateType, boolean directDownload) throws ResourceAllocationException { + TemplateType templateType, boolean directDownload, boolean deployAsIs) throws ResourceAllocationException { //Long accountId = null; // parameters verification @@ -257,7 +257,7 @@ public abstract class TemplateAdapterBase extends AdapterBase implements Templat CallContext.current().setEventDetails("Id: " + id + " name: " + name); return new TemplateProfile(id, userId, name, displayText, bits, passwordEnabled, requiresHVM, url, isPublic, featured, isExtractable, imgfmt, guestOSId, zoneIdList, hypervisorType, templateOwner.getAccountName(), templateOwner.getDomainId(), templateOwner.getAccountId(), chksum, bootable, templateTag, details, - sshkeyEnabled, null, isDynamicallyScalable, templateType, directDownload); + sshkeyEnabled, null, isDynamicallyScalable, templateType, directDownload, deployAsIs); } @@ -285,7 +285,8 @@ public abstract class TemplateAdapterBase extends AdapterBase implements Templat return prepare(false, CallContext.current().getCallingUserId(), cmd.getTemplateName(), cmd.getDisplayText(), cmd.getBits(), cmd.isPasswordEnabled(), cmd.getRequiresHvm(), cmd.getUrl(), cmd.isPublic(), cmd.isFeatured(), cmd.isExtractable(), cmd.getFormat(), cmd.getOsTypeId(), zoneId, hypervisorType, cmd.getChecksum(), true, - cmd.getTemplateTag(), owner, cmd.getDetails(), cmd.isSshKeyEnabled(), null, cmd.isDynamicallyScalable(), isRouting ? TemplateType.ROUTING : TemplateType.USER, cmd.isDirectDownload()); + cmd.getTemplateTag(), owner, cmd.getDetails(), cmd.isSshKeyEnabled(), null, cmd.isDynamicallyScalable(), isRouting ? TemplateType.ROUTING : TemplateType.USER, + cmd.isDirectDownload(), cmd.isDeployAsIs()); } @@ -316,7 +317,7 @@ public abstract class TemplateAdapterBase extends AdapterBase implements Templat params.isExtractable(), params.getFormat(), params.getGuestOSId(), zoneList, params.getHypervisorType(), params.getChecksum(), params.isBootable(), params.getTemplateTag(), owner, params.getDetails(), params.isSshKeyEnabled(), params.getImageStoreUuid(), - params.isDynamicallyScalable(), params.isRoutingType() ? TemplateType.ROUTING : TemplateType.USER, params.isDirectDownload()); + params.isDynamicallyScalable(), params.isRoutingType() ? TemplateType.ROUTING : TemplateType.USER, params.isDirectDownload(), false); } @Override @@ -358,7 +359,7 @@ public abstract class TemplateAdapterBase extends AdapterBase implements Templat return prepare(true, CallContext.current().getCallingUserId(), cmd.getIsoName(), cmd.getDisplayText(), 64, cmd.isPasswordEnabled(), true, cmd.getUrl(), cmd.isPublic(), cmd.isFeatured(), cmd.isExtractable(), ImageFormat.ISO.toString(), cmd.getOsTypeId(), zoneList, HypervisorType.None, cmd.getChecksum(), cmd.isBootable(), null, - owner, null, false, cmd.getImageStoreUuid(), cmd.isDynamicallyScalable(), TemplateType.USER, cmd.isDirectDownload()); + owner, null, false, cmd.getImageStoreUuid(), cmd.isDynamicallyScalable(), TemplateType.USER, cmd.isDirectDownload(), false); } protected VMTemplateVO persistTemplate(TemplateProfile profile, VirtualMachineTemplate.State initialState) { @@ -367,7 +368,7 @@ public abstract class TemplateAdapterBase extends AdapterBase implements Templat new VMTemplateVO(profile.getTemplateId(), profile.getName(), profile.getFormat(), profile.isPublic(), profile.isFeatured(), profile.isExtractable(), profile.getTemplateType(), profile.getUrl(), profile.isRequiresHVM(), profile.getBits(), profile.getAccountId(), profile.getCheckSum(), profile.getDisplayText(), profile.isPasswordEnabled(), profile.getGuestOsId(), profile.isBootable(), profile.getHypervisorType(), - profile.getTemplateTag(), profile.getDetails(), profile.isSshKeyEnabled(), profile.IsDynamicallyScalable(), profile.isDirectDownload()); + profile.getTemplateTag(), profile.getDetails(), profile.isSshKeyEnabled(), profile.IsDynamicallyScalable(), profile.isDirectDownload(), profile.isDeployAsIs()); template.setState(initialState); if (profile.isDirectDownload()) { diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index 3a241faccd9..c58b2f1ad9a 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -32,6 +32,7 @@ import java.util.concurrent.Executors; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.agent.api.to.DatadiskTO; import org.apache.cloudstack.acl.SecurityChecker.AccessType; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseListTemplateOrIsoPermissionsCmd; @@ -669,7 +670,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, VMTemplateStoragePoolVO templateStoragePoolRef = null; TemplateDataStoreVO templateStoreRef = null; - templateStoragePoolRef = _tmpltPoolDao.findByPoolTemplate(poolId, templateId); + templateStoragePoolRef = _tmpltPoolDao.findByPoolTemplate(poolId, templateId, null); if (templateStoragePoolRef != null) { templateStoragePoolRef.setMarkedForGC(false); _tmpltPoolDao.update(templateStoragePoolRef.getId(), templateStoragePoolRef); @@ -709,7 +710,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, return null; } - return _tmpltPoolDao.findByPoolTemplate(poolId, templateId); + return _tmpltPoolDao.findByPoolTemplate(poolId, templateId, null); } catch (Exception ex) { s_logger.debug("failed to copy template from image store:" + srcSecStore.getName() + " to primary storage"); } @@ -1021,7 +1022,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, } PrimaryDataStore pool = (PrimaryDataStore)_dataStoreMgr.getPrimaryDataStore(templatePoolVO.getPoolId()); - TemplateInfo template = _tmplFactory.getTemplate(templatePoolRef.getTemplateId(), pool); + TemplateInfo template = _tmplFactory.getTemplateOnPrimaryStorage(templatePoolRef.getTemplateId(), pool, templatePoolRef.getDeploymentOption()); try { if (s_logger.isDebugEnabled()) { @@ -1897,7 +1898,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, } privateTemplate = new VMTemplateVO(nextTemplateId, name, ImageFormat.RAW, isPublic, featured, isExtractable, TemplateType.USER, null, requiresHvmValue, bitsValue, templateOwner.getId(), null, description, - passwordEnabledValue, guestOS.getId(), true, hyperType, templateTag, cmd.getDetails(), sshKeyEnabledValue, isDynamicScalingEnabled, false); + passwordEnabledValue, guestOS.getId(), true, hyperType, templateTag, cmd.getDetails(), sshKeyEnabledValue, isDynamicScalingEnabled, false, false); if (sourceTemplateId != null) { if (s_logger.isDebugEnabled()) { @@ -2223,4 +2224,16 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, public void setTemplateAdapters(List adapters) { _adapters = adapters; } + + @Override + public List getTemplateDisksOnImageStore(Long templateId, DataStoreRole role, String configurationId) { + TemplateInfo templateObject = _tmplFactory.getTemplate(templateId, role); + if (templateObject == null) { + String msg = String.format("Could not find template %s downloaded on store with role %s", templateId, role.toString()); + s_logger.error(msg); + throw new CloudRuntimeException(msg); + } + return _tmpltSvr.getTemplateDatadisksOnImageStore(templateObject, configurationId); + } + } diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 525931deddb..a7f01cb1be3 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -47,8 +47,12 @@ import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; +import com.cloud.agent.api.storage.OVFPropertyTO; import com.cloud.exception.UnsupportedServiceException; import com.cloud.hypervisor.Hypervisor; +import com.cloud.storage.ImageStore; +import com.cloud.storage.VMTemplateDetailVO; +import com.cloud.storage.dao.VMTemplateDetailsDao; import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.acl.SecurityChecker.AccessType; import org.apache.cloudstack.affinity.AffinityGroupService; @@ -79,6 +83,7 @@ import org.apache.cloudstack.api.command.user.vm.UpgradeVMCmd; import org.apache.cloudstack.api.command.user.vmgroup.CreateVMGroupCmd; import org.apache.cloudstack.api.command.user.vmgroup.DeleteVMGroupCmd; import org.apache.cloudstack.api.command.user.volume.ResizeVolumeCmd; +import org.apache.cloudstack.api.net.NetworkPrerequisiteTO; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.cloud.entity.api.VirtualMachineEntity; import org.apache.cloudstack.engine.cloud.entity.api.db.dao.VMNetworkMapDao; @@ -105,6 +110,7 @@ import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import org.apache.commons.codec.binary.Base64; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; @@ -257,7 +263,6 @@ import com.cloud.storage.Storage.StoragePoolType; import com.cloud.storage.Storage.TemplateType; import com.cloud.storage.StoragePool; import com.cloud.storage.StoragePoolStatus; -import com.cloud.storage.TemplateOVFPropertyVO; import com.cloud.storage.VMTemplateStorageResourceAssoc; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.VMTemplateZoneVO; @@ -268,7 +273,6 @@ import com.cloud.storage.dao.DiskOfferingDao; import com.cloud.storage.dao.GuestOSCategoryDao; import com.cloud.storage.dao.GuestOSDao; import com.cloud.storage.dao.SnapshotDao; -import com.cloud.storage.dao.TemplateOVFPropertiesDao; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VMTemplateZoneDao; import com.cloud.storage.dao.VolumeDao; @@ -503,7 +507,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir @Inject private ResourceTagDao resourceTagDao; @Inject - private TemplateOVFPropertiesDao templateOVFPropertiesDao; + private VMTemplateDetailsDao templateDetailsDao; private ScheduledExecutorService _executor = null; private ScheduledExecutorService _vmIpFetchExecutor = null; @@ -2500,10 +2504,12 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir } } for (String detailName : details.keySet()) { - if (detailName.startsWith(ApiConstants.OVF_PROPERTIES)) { - String ovfPropKey = detailName.replace(ApiConstants.OVF_PROPERTIES + "-", ""); - TemplateOVFPropertyVO ovfPropertyVO = templateOVFPropertiesDao.findByTemplateAndKey(vmInstance.getTemplateId(), ovfPropKey); - if (ovfPropertyVO != null && ovfPropertyVO.isPassword()) { + if (s_logger.isTraceEnabled()) { + s_logger.trace(String.format("looking for vm detail '%s'", detailName)); + } + if (detailName.startsWith(ImageStore.ACS_PROPERTY_PREFIX)) { + OVFPropertyTO propertyTO = templateDetailsDao.findPropertyByTemplateAndKey(vmInstance.getTemplateId(),detailName); + if (propertyTO != null && propertyTO.isPassword()) { details.put(detailName, DBEncryptionUtil.encrypt(details.get(detailName))); } } @@ -3307,56 +3313,10 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir List vpcSupportedHTypes = _vpcMgr.getSupportedVpcHypervisors(); if (networkIdList == null || networkIdList.isEmpty()) { - NetworkVO defaultNetwork = null; - - // if no network is passed in - // Check if default virtual network offering has - // Availability=Required. If it's true, search for corresponding - // network - // * if network is found, use it. If more than 1 virtual network is - // found, throw an error - // * if network is not found, create a new one and use it - - List requiredOfferings = _networkOfferingDao.listByAvailability(Availability.Required, false); - if (requiredOfferings.size() < 1) { - throw new InvalidParameterValueException("Unable to find network offering with availability=" + Availability.Required - + " to automatically create the network as a part of vm creation"); - } - - if (requiredOfferings.get(0).getState() == NetworkOffering.State.Enabled) { - // get Virtual networks - List virtualNetworks = _networkModel.listNetworksForAccount(owner.getId(), zone.getId(), Network.GuestType.Isolated); - if (virtualNetworks == null) { - throw new InvalidParameterValueException("No (virtual) networks are found for account " + owner); - } - if (virtualNetworks.isEmpty()) { - long physicalNetworkId = _networkModel.findPhysicalNetworkId(zone.getId(), requiredOfferings.get(0).getTags(), requiredOfferings.get(0).getTrafficType()); - // Validate physical network - PhysicalNetwork physicalNetwork = _physicalNetworkDao.findById(physicalNetworkId); - if (physicalNetwork == null) { - throw new InvalidParameterValueException("Unable to find physical network with id: " + physicalNetworkId + " and tag: " - + requiredOfferings.get(0).getTags()); - } - s_logger.debug("Creating network for account " + owner + " from the network offering id=" + requiredOfferings.get(0).getId() + " as a part of deployVM process"); - Network newNetwork = _networkMgr.createGuestNetwork(requiredOfferings.get(0).getId(), owner.getAccountName() + "-network", owner.getAccountName() + "-network", - null, null, null, false, null, owner, null, physicalNetwork, zone.getId(), ACLType.Account, null, null, null, null, true, null, null, - null); - if (newNetwork != null) { - defaultNetwork = _networkDao.findById(newNetwork.getId()); - } - } else if (virtualNetworks.size() > 1) { - throw new InvalidParameterValueException("More than 1 default Isolated networks are found for account " + owner + "; please specify networkIds"); - } else { - defaultNetwork = _networkDao.findById(virtualNetworks.get(0).getId()); - } - } else { - throw new InvalidParameterValueException("Required network offering id=" + requiredOfferings.get(0).getId() + " is not in " + NetworkOffering.State.Enabled); - } - + NetworkVO defaultNetwork = getDefaultNetwork(zone, owner, false); if (defaultNetwork != null) { networkList.add(defaultNetwork); } - } else { for (Long networkId : networkIdList) { NetworkVO network = _networkDao.findById(networkId); @@ -3393,6 +3353,91 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir dataDiskTemplateToDiskOfferingMap, userVmOVFPropertiesMap); } + private NetworkVO getNetworkToAddToNetworkList(VirtualMachineTemplate template, Account owner, HypervisorType hypervisor, + List vpcSupportedHTypes, Long networkId) { + NetworkVO network = _networkDao.findById(networkId); + if (network == null) { + throw new InvalidParameterValueException("Unable to find network by id " + networkId); + } + if (network.getVpcId() != null) { + // Only ISOs, XenServer, KVM, and VmWare template types are + // supported for vpc networks + if (template.getFormat() != ImageFormat.ISO && !vpcSupportedHTypes.contains(template.getHypervisorType())) { + throw new InvalidParameterValueException("Can't create vm from template with hypervisor " + template.getHypervisorType() + " in vpc network " + network); + } else if (template.getFormat() == ImageFormat.ISO && !vpcSupportedHTypes.contains(hypervisor)) { + // Only XenServer, KVM, and VMware hypervisors are supported + // for vpc networks + throw new InvalidParameterValueException("Can't create vm of hypervisor type " + hypervisor + " in vpc network"); + } + } + + _networkModel.checkNetworkPermissions(owner, network); + + // don't allow to use system networks + NetworkOffering networkOffering = _entityMgr.findById(NetworkOffering.class, network.getNetworkOfferingId()); + if (networkOffering.isSystemOnly()) { + throw new InvalidParameterValueException("Network id=" + networkId + " is system only and can't be used for vm deployment"); + } + return network; + } + + private NetworkVO getDefaultNetwork(DataCenter zone, Account owner, boolean selectAny) throws InsufficientCapacityException, ResourceAllocationException { + NetworkVO defaultNetwork = null; + + // if no network is passed in + // Check if default virtual network offering has + // Availability=Required. If it's true, search for corresponding + // network + // * if network is found, use it. If more than 1 virtual network is + // found, throw an error + // * if network is not found, create a new one and use it + + List requiredOfferings = _networkOfferingDao.listByAvailability(Availability.Required, false); + if (requiredOfferings.size() < 1) { + throw new InvalidParameterValueException("Unable to find network offering with availability=" + Availability.Required + + " to automatically create the network as a part of vm creation"); + } + + if (requiredOfferings.get(0).getState() == NetworkOffering.State.Enabled) { + // get Virtual networks + List virtualNetworks = _networkModel.listNetworksForAccount(owner.getId(), zone.getId(), Network.GuestType.Isolated); + if (virtualNetworks == null) { + throw new InvalidParameterValueException("No (virtual) networks are found for account " + owner); + } + if (virtualNetworks.isEmpty()) { + defaultNetwork = createDefaultNetworkForAccount(zone, owner, requiredOfferings); + } else if (virtualNetworks.size() > 1 && !selectAny) { + throw new InvalidParameterValueException("More than 1 default Isolated networks are found for account " + owner + "; please specify networkIds"); + } else { + defaultNetwork = _networkDao.findById(virtualNetworks.get(0).getId()); + } + } else { + throw new InvalidParameterValueException("Required network offering id=" + requiredOfferings.get(0).getId() + " is not in " + NetworkOffering.State.Enabled); + } + + return defaultNetwork; + } + + private NetworkVO createDefaultNetworkForAccount(DataCenter zone, Account owner, List requiredOfferings) + throws InsufficientCapacityException, ResourceAllocationException { + NetworkVO defaultNetwork = null; + long physicalNetworkId = _networkModel.findPhysicalNetworkId(zone.getId(), requiredOfferings.get(0).getTags(), requiredOfferings.get(0).getTrafficType()); + // Validate physical network + PhysicalNetwork physicalNetwork = _physicalNetworkDao.findById(physicalNetworkId); + if (physicalNetwork == null) { + throw new InvalidParameterValueException("Unable to find physical network with id: " + physicalNetworkId + " and tag: " + + requiredOfferings.get(0).getTags()); + } + s_logger.debug("Creating network for account " + owner + " from the network offering id=" + requiredOfferings.get(0).getId() + " as a part of deployVM process"); + Network newNetwork = _networkMgr.createGuestNetwork(requiredOfferings.get(0).getId(), owner.getAccountName() + "-network", owner.getAccountName() + "-network", + null, null, null, false, null, owner, null, physicalNetwork, zone.getId(), ACLType.Account, null, null, null, null, true, null, null, + null); + if (newNetwork != null) { + defaultNetwork = _networkDao.findById(newNetwork.getId()); + } + return defaultNetwork; + } + private void verifyExtraDhcpOptionsNetwork(Map> dhcpOptionsMap, List networkList) throws InvalidParameterValueException { if (dhcpOptionsMap != null) { for (String networkUuid : dhcpOptionsMap.keySet()) { @@ -3646,12 +3691,11 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir sshPublicKey = pair.getPublicKey(); } - List> networks = new ArrayList>(); - - LinkedHashMap networkNicMap = new LinkedHashMap(); + LinkedHashMap> networkNicMap = new LinkedHashMap<>(); short defaultNetworkNumber = 0; boolean securityGroupEnabled = false; + int networkIndex = 0; for (NetworkVO network : networkList) { if ((network.getDataCenterId() != zone.getId())) { if (!network.isStrechedL2Network()) { @@ -3689,7 +3733,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir } NicProfile profile = new NicProfile(requestedIpPair.getIp4Address(), requestedIpPair.getIp6Address(), requestedIpPair.getMacAddress()); - + profile.setOrderIndex(networkIndex); if (defaultNetworkNumber == 0) { defaultNetworkNumber++; // if user requested specific ip for default network, add it @@ -3716,13 +3760,16 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir } } - networks.add(new Pair(network, profile)); - if (_networkModel.isSecurityGroupSupportedInNetwork(network)) { securityGroupEnabled = true; } - - networkNicMap.put(network.getUuid(), profile); + List profiles = networkNicMap.get(network.getUuid()); + if (CollectionUtils.isEmpty(profiles)) { + profiles = new ArrayList<>(); + } + profiles.add(profile); + networkNicMap.put(network.getUuid(), profiles); + networkIndex++; } if (securityGroupIdList != null && !securityGroupIdList.isEmpty() && !securityGroupEnabled) { @@ -3853,7 +3900,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir private UserVmVO commitUserVm(final boolean isImport, final DataCenter zone, final Host host, final Host lastHost, final VirtualMachineTemplate template, final String hostName, final String displayName, final Account owner, final Long diskOfferingId, final Long diskSize, final String userData, final Account caller, final Boolean isDisplayVm, final String keyboard, - final long accountId, final long userId, final ServiceOffering offering, final boolean isIso, final String sshPublicKey, final LinkedHashMap networkNicMap, + final long accountId, final long userId, final ServiceOffering offering, final boolean isIso, final String sshPublicKey, final LinkedHashMap> networkNicMap, final long id, final String instanceName, final String uuidName, final HypervisorType hypervisorType, final Map customParameters, final Map> extraDhcpOptionMap, final Map dataDiskTemplateToDiskOfferingMap, final Map userVmOVFPropertiesMap, final VirtualMachine.PowerState powerState) throws InsufficientCapacityException { @@ -3960,27 +4007,11 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir } vm.setDetail(VmDetailConstants.DEPLOY_VM, "true"); - if (MapUtils.isNotEmpty(userVmOVFPropertiesMap)) { - for (String key : userVmOVFPropertiesMap.keySet()) { - String detailKey = ApiConstants.OVF_PROPERTIES + "-" + key; - String value = userVmOVFPropertiesMap.get(key); + copyDiskDetailsToVm(vm, template); - // Sanitize boolean values to expected format and encrypt passwords - if (StringUtils.isNotBlank(value)) { - if (value.equalsIgnoreCase("True")) { - value = "True"; - } else if (value.equalsIgnoreCase("False")) { - value = "False"; - } else { - TemplateOVFPropertyVO ovfPropertyVO = templateOVFPropertiesDao.findByTemplateAndKey(vm.getTemplateId(), key); - if (ovfPropertyVO.isPassword()) { - value = DBEncryptionUtil.encrypt(value); - } - } - } - vm.setDetail(detailKey, value); - } - } + setPropertiesOnVM(vm, userVmOVFPropertiesMap); + + copyNetworkRequirementsToVm(vm, template); _vmDao.saveDetails(vm); if (!isImport) { @@ -4025,9 +4056,59 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir }); } + private void copyNetworkRequirementsToVm(UserVmVO vm, VirtualMachineTemplate template) { + if (template.isDeployAsIs()) { + List details = templateDetailsDao.listDetailsByTemplateIdMatchingPrefix(template.getId(), ImageStore.REQUIRED_NETWORK_PREFIX); + for (VMTemplateDetailVO detail : details) { + vm.setDetail(detail.getName(), detail.getValue()); + } + } + } + + private void copyDiskDetailsToVm(UserVmVO vm, VirtualMachineTemplate template) { + if (template.isDeployAsIs()) { + List details = templateDetailsDao.listDetailsByTemplateIdMatchingPrefix(template.getId(), ImageStore.DISK_DEFINITION_PREFIX); + for (VMTemplateDetailVO detail : details) { + vm.setDetail(detail.getName(), detail.getValue()); + } + } + } + + /** + * take the properties and set them on the vm. + * consider should we be complete, and make sure all default values are copied as well if known? + * I.E. iterate over the template details as well to copy any that are not defined yet. + */ + private void setPropertiesOnVM(UserVmVO vm, Map userVmOVFPropertiesMap) { + if (MapUtils.isNotEmpty(userVmOVFPropertiesMap)) { + for (String key : userVmOVFPropertiesMap.keySet()) { + String detailKey = ImageStore.ACS_PROPERTY_PREFIX + key; + String value = userVmOVFPropertiesMap.get(key); + + // Sanitize boolean values to expected format and encrypt passwords + if (StringUtils.isNotBlank(value)) { + if (value.equalsIgnoreCase("True")) { + value = "True"; + } else if (value.equalsIgnoreCase("False")) { + value = "False"; + } else { + OVFPropertyTO propertyTO = templateDetailsDao.findPropertyByTemplateAndKey(vm.getTemplateId(), key); + if (propertyTO != null && propertyTO.isPassword()) { + value = DBEncryptionUtil.encrypt(value); + } + } + } + if (s_logger.isTraceEnabled()) { + s_logger.trace(String.format("setting property '%s' as '%s' with value '%s'", key, detailKey, value)); + } + vm.setDetail(detailKey, value); + } + } + } + private UserVmVO commitUserVm(final DataCenter zone, final VirtualMachineTemplate template, final String hostName, final String displayName, final Account owner, final Long diskOfferingId, final Long diskSize, final String userData, final Account caller, final Boolean isDisplayVm, final String keyboard, - final long accountId, final long userId, final ServiceOfferingVO offering, final boolean isIso, final String sshPublicKey, final LinkedHashMap networkNicMap, + final long accountId, final long userId, final ServiceOfferingVO offering, final boolean isIso, final String sshPublicKey, final LinkedHashMap> networkNicMap, final long id, final String instanceName, final String uuidName, final HypervisorType hypervisorType, final Map customParameters, final Map> extraDhcpOptionMap, final Map dataDiskTemplateToDiskOfferingMap, Map userVmOVFPropertiesMap) throws InsufficientCapacityException { @@ -5143,6 +5224,12 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir } } + List networkIds = cmd.getNetworkIds(); + LinkedHashMap userVmNetworkMap = getVmOvfNetworkMapping(zone, owner, template, cmd.getVmNetworkMap()); + if (MapUtils.isNotEmpty(userVmNetworkMap)) { + networkIds = new ArrayList<>(userVmNetworkMap.values()); + } + String ipAddress = cmd.getIpAddress(); String ip6Address = cmd.getIp6Address(); String macAddress = cmd.getMacAddress(); @@ -5157,9 +5244,9 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir Boolean displayVm = cmd.isDisplayVm(); String keyboard = cmd.getKeyboard(); Map dataDiskTemplateToDiskOfferingMap = cmd.getDataDiskTemplateToDiskOfferingMap(); - Map userVmOVFProperties = cmd.getVmOVFProperties(); + Map userVmOVFProperties = cmd.getVmProperties(); if (zone.getNetworkType() == NetworkType.Basic) { - if (cmd.getNetworkIds() != null) { + if (networkIds != null) { throw new InvalidParameterValueException("Can't specify network Ids in Basic zone"); } else { vm = createBasicSecurityGroupVirtualMachine(zone, serviceOffering, template, getSecurityGroupIdList(cmd), owner, name, displayName, diskOfferingId, @@ -5169,7 +5256,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir } } else { if (zone.isSecurityGroupEnabled()) { - vm = createAdvancedSecurityGroupVirtualMachine(zone, serviceOffering, template, cmd.getNetworkIds(), getSecurityGroupIdList(cmd), owner, name, + vm = createAdvancedSecurityGroupVirtualMachine(zone, serviceOffering, template, networkIds, getSecurityGroupIdList(cmd), owner, name, displayName, diskOfferingId, size, group, cmd.getHypervisor(), cmd.getHttpMethod(), userData, sshKeyPairName, cmd.getIpToNetworkMap(), addrs, displayVm, keyboard, cmd.getAffinityGroupIdList(), cmd.getDetails(), cmd.getCustomId(), cmd.getDhcpOptionsMap(), dataDiskTemplateToDiskOfferingMap, userVmOVFProperties); @@ -5178,7 +5265,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir if (cmd.getSecurityGroupIdList() != null && !cmd.getSecurityGroupIdList().isEmpty()) { throw new InvalidParameterValueException("Can't create vm with security groups; security group feature is not enabled per zone"); } - vm = createAdvancedVirtualMachine(zone, serviceOffering, template, cmd.getNetworkIds(), owner, name, displayName, diskOfferingId, size, group, + vm = createAdvancedVirtualMachine(zone, serviceOffering, template, networkIds, owner, name, displayName, diskOfferingId, size, group, cmd.getHypervisor(), cmd.getHttpMethod(), userData, sshKeyPairName, cmd.getIpToNetworkMap(), addrs, displayVm, keyboard, cmd.getAffinityGroupIdList(), cmd.getDetails(), cmd.getCustomId(), cmd.getDhcpOptionsMap(), dataDiskTemplateToDiskOfferingMap, userVmOVFProperties); } @@ -7287,4 +7374,42 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir } } } -} \ No newline at end of file + + private LinkedHashMap getVmOvfNetworkMapping(DataCenter zone, Account owner, VirtualMachineTemplate template, Map vmNetworkMapping) throws InsufficientCapacityException, ResourceAllocationException { + LinkedHashMap mapping = new LinkedHashMap<>(); + if (ImageFormat.OVA.equals(template.getFormat())) { + List networkPrerequisiteTOList = + templateDetailsDao.listNetworkRequirementsByTemplateId(template.getId()); + if (CollectionUtils.isNotEmpty(networkPrerequisiteTOList)) { + Network lastMappedNetwork = null; + for (NetworkPrerequisiteTO networkPrerequisiteTO : networkPrerequisiteTOList) { + Long networkId = vmNetworkMapping.get(networkPrerequisiteTO.getInstanceID()); + if (networkId == null && lastMappedNetwork == null) { + lastMappedNetwork = getNetworkForOvfNetworkMapping(zone, owner); + } + if (networkId == null) { + networkId = lastMappedNetwork.getId(); + } + mapping.put(networkPrerequisiteTO.getInstanceID(), networkId); + } + } + } + return mapping; + } + + private Network getNetworkForOvfNetworkMapping(DataCenter zone, Account owner) throws InsufficientCapacityException, ResourceAllocationException { + Network network = null; + if (zone.isSecurityGroupEnabled()) { + network = _networkModel.getNetworkWithSGWithFreeIPs(zone.getId()); + if (network == null) { + throw new InvalidParameterValueException("No network with security enabled is found in zone ID: " + zone.getUuid()); + } + } else { + network = getDefaultNetwork(zone, owner, true); + if (network == null) { + throw new InvalidParameterValueException(String.format("Default network not found for zone ID: %s and account ID: %s", zone.getUuid(), owner.getUuid())); + } + } + return network; + } +} diff --git a/server/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java b/server/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java index a05c4b9e4aa..dcbc9656448 100644 --- a/server/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/direct/download/DirectDownloadManagerImpl.java @@ -263,13 +263,13 @@ public class DirectDownloadManagerImpl extends ManagerBase implements DirectDown Answer answer = sendDirectDownloadCommand(cmd, template, poolId, host); - VMTemplateStoragePoolVO sPoolRef = vmTemplatePoolDao.findByPoolTemplate(poolId, templateId); + VMTemplateStoragePoolVO sPoolRef = vmTemplatePoolDao.findByPoolTemplate(poolId, templateId, null); if (sPoolRef == null) { if (s_logger.isDebugEnabled()) { s_logger.debug("Not found (templateId:" + templateId + " poolId: " + poolId + ") in template_spool_ref, persisting it"); } DirectDownloadAnswer ans = (DirectDownloadAnswer) answer; - sPoolRef = new VMTemplateStoragePoolVO(poolId, templateId); + sPoolRef = new VMTemplateStoragePoolVO(poolId, templateId, null); sPoolRef.setDownloadPercent(100); sPoolRef.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOADED); sPoolRef.setState(ObjectInDataStoreStateMachine.State.Ready); diff --git a/server/src/test/java/com/cloud/api/query/dao/TemplateJoinDaoImplTest.java b/server/src/test/java/com/cloud/api/query/dao/TemplateJoinDaoImplTest.java index a6b33edd1aa..d4cbf910c24 100755 --- a/server/src/test/java/com/cloud/api/query/dao/TemplateJoinDaoImplTest.java +++ b/server/src/test/java/com/cloud/api/query/dao/TemplateJoinDaoImplTest.java @@ -115,5 +115,4 @@ public class TemplateJoinDaoImplTest extends GenericDaoBaseWithTagInformationBas ReflectionTestUtils.setField(template, "detailName", detailName); ReflectionTestUtils.setField(template, "detailValue", detailValue); } - } \ No newline at end of file diff --git a/server/src/test/java/com/cloud/template/TemplateManagerImplTest.java b/server/src/test/java/com/cloud/template/TemplateManagerImplTest.java index bd21643b4a1..47180cda0a1 100755 --- a/server/src/test/java/com/cloud/template/TemplateManagerImplTest.java +++ b/server/src/test/java/com/cloud/template/TemplateManagerImplTest.java @@ -119,6 +119,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.nullable; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyBoolean; import static org.mockito.Matchers.anyLong; @@ -287,7 +288,7 @@ public class TemplateManagerImplTest { when(dataStoreManager.getPrimaryDataStore(anyLong())).thenReturn(mockPrimaryDataStore); when(vmTemplateDao.findById(anyLong(), anyBoolean())).thenReturn(mockTemplate); - when(vmTemplatePoolDao.findByPoolTemplate(anyLong(), anyLong())).thenReturn(mockTemplateStore); + when(vmTemplatePoolDao.findByPoolTemplate(anyLong(), anyLong(), nullable(String.class))).thenReturn(mockTemplateStore); doNothing().when(mockTemplateStore).setMarkedForGC(anyBoolean()); @@ -309,7 +310,7 @@ public class TemplateManagerImplTest { when(dataStoreManager.getPrimaryDataStore(anyLong())).thenReturn(mockPrimaryDataStore); when(vmTemplateDao.findById(anyLong(), anyBoolean())).thenReturn(mockTemplate); - when(vmTemplatePoolDao.findByPoolTemplate(anyLong(), anyLong())).thenReturn(null); + when(vmTemplatePoolDao.findByPoolTemplate(anyLong(), anyLong(), nullable(String.class))).thenReturn(null); when(templateDataStoreDao.findByTemplateZoneDownloadStatus(202l, 1l, VMTemplateStorageResourceAssoc.Status.DOWNLOADED)).thenReturn(null); VMTemplateStoragePoolVO returnObject = templateManager.prepareTemplateForCreate(mockTemplate, (StoragePool) mockPrimaryDataStore); @@ -332,7 +333,7 @@ public class TemplateManagerImplTest { when(dataStoreManager.getPrimaryDataStore(anyLong())).thenReturn(mockPrimaryDataStore); when(vmTemplateDao.findById(anyLong(), anyBoolean())).thenReturn(mockTemplate); - when(vmTemplatePoolDao.findByPoolTemplate(anyLong(), anyLong())).thenReturn(null); + when(vmTemplatePoolDao.findByPoolTemplate(anyLong(), anyLong(), nullable(String.class))).thenReturn(null); when(templateDataStoreDao.findByTemplateZoneDownloadStatus(202l, 1l, VMTemplateStorageResourceAssoc.Status.DOWNLOADED)).thenReturn(mockTemplateDataStore); when(storagePoolHostDao.listByHostStatus(2l, Status.Up)).thenReturn(null); @@ -361,7 +362,7 @@ public class TemplateManagerImplTest { when(vmTemplateDao.findById(anyLong())).thenReturn(mockTemplate); when(dataStoreManager.getPrimaryDataStore(anyLong())).thenReturn(mockPrimaryDataStore); when(vmTemplateDao.findById(anyLong(), anyBoolean())).thenReturn(mockTemplate); - when(vmTemplatePoolDao.findByPoolTemplate(anyLong(), anyLong())).thenReturn(mockTemplateStore); + when(vmTemplatePoolDao.findByPoolTemplate(anyLong(), anyLong(), nullable(String.class))).thenReturn(mockTemplateStore); when(primaryDataStoreDao.findById(anyLong())).thenReturn(mockPool); doNothing().when(mockTemplateStore).setMarkedForGC(anyBoolean()); @@ -390,7 +391,7 @@ public class TemplateManagerImplTest { when(vmTemplateDao.findById(anyLong())).thenReturn(mockTemplate); when(dataStoreManager.getPrimaryDataStore(anyLong())).thenReturn(mockPrimaryDataStore); when(vmTemplateDao.findById(anyLong(), anyBoolean())).thenReturn(mockTemplate); - when(vmTemplatePoolDao.findByPoolTemplate(anyLong(), anyLong())).thenReturn(mockTemplateStore); + when(vmTemplatePoolDao.findByPoolTemplate(anyLong(), anyLong(), nullable(String.class))).thenReturn(mockTemplateStore); when(primaryDataStoreDao.findById(anyLong())).thenReturn(mockPool); doNothing().when(mockTemplateStore).setMarkedForGC(anyBoolean()); @@ -432,7 +433,7 @@ public class TemplateManagerImplTest { when(vmTemplateDao.findById(anyLong())).thenReturn(mockTemplate); when(dataStoreManager.getPrimaryDataStore(anyLong())).thenReturn(mockPrimaryDataStore); when(vmTemplateDao.findById(anyLong(), anyBoolean())).thenReturn(mockTemplate); - when(vmTemplatePoolDao.findByPoolTemplate(anyLong(), anyLong())).thenReturn(mockTemplateStore); + when(vmTemplatePoolDao.findByPoolTemplate(anyLong(), anyLong(), nullable(String.class))).thenReturn(mockTemplateStore); when(primaryDataStoreDao.findById(2l)).thenReturn(mockPool1); when(primaryDataStoreDao.findById(3l)).thenReturn(mockPool2); when(primaryDataStoreDao.findById(4l)).thenReturn(mockPool3); diff --git a/server/src/test/java/com/cloud/vm/DeploymentPlanningManagerImplTest.java b/server/src/test/java/com/cloud/vm/DeploymentPlanningManagerImplTest.java index e73c0c6bd7d..d356570b633 100644 --- a/server/src/test/java/com/cloud/vm/DeploymentPlanningManagerImplTest.java +++ b/server/src/test/java/com/cloud/vm/DeploymentPlanningManagerImplTest.java @@ -29,6 +29,8 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; import com.cloud.host.Host; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.dao.VMTemplateDao; import org.apache.cloudstack.affinity.dao.AffinityGroupDomainMapDao; import org.junit.Before; import org.junit.BeforeClass; @@ -141,6 +143,9 @@ public class DeploymentPlanningManagerImplTest { @Inject UserVmDetailsDao vmDetailsDao; + @Inject + VMTemplateDao templateDao; + @Mock Host host; @@ -162,6 +167,10 @@ public class DeploymentPlanningManagerImplTest { Mockito.when(_plannerHostReserveDao.findById(Matchers.anyLong())).thenReturn(reservationVO); Mockito.when(_affinityGroupVMMapDao.countAffinityGroupsForVm(Matchers.anyLong())).thenReturn(0L); + VMTemplateVO template = Mockito.mock(VMTemplateVO.class); + Mockito.when(template.isDeployAsIs()).thenReturn(false); + Mockito.when(templateDao.findById(Mockito.anyLong())).thenReturn(template); + VMInstanceVO vm = new VMInstanceVO(); Mockito.when(vmProfile.getVirtualMachine()).thenReturn(vm); @@ -456,6 +465,11 @@ public class DeploymentPlanningManagerImplTest { return Mockito.mock(HostGpuGroupsDao.class); } + @Bean + public VMTemplateDao vmTemplateDao() { + return Mockito.mock(VMTemplateDao.class); + } + public static class Library implements TypeFilter { @Override diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index 863376d3381..03eba4a6b84 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@ -190,6 +190,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S private static final String TEMPLATE_ROOT_DIR = "template/tmpl"; private static final String VOLUME_ROOT_DIR = "volumes"; private static final String POST_UPLOAD_KEY_LOCATION = "/etc/cloudstack/agent/ms-psk"; + private static final String ORIGINAL_FILE_EXTENSION = ".orig"; private static final Map updatableConfigData = Maps.newHashMap(); static { @@ -391,6 +392,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S public Answer execute(GetDatadisksCommand cmd) { DataTO srcData = cmd.getData(); + String configurationId = cmd.getConfigurationId(); TemplateObjectTO template = (TemplateObjectTO)srcData; DataStoreTO srcStore = srcData.getDataStore(); if (!(srcStore instanceof NfsTO)) { @@ -435,7 +437,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S Script command = new Script("cp", _timeout, s_logger); command.add(ovfFilePath); - command.add(ovfFilePath + ".orig"); + command.add(ovfFilePath + ORIGINAL_FILE_EXTENSION); String result = command.execute(); if (result != null) { String msg = "Unable to rename original OVF, error msg: " + result; @@ -445,7 +447,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S s_logger.debug("Reading OVF " + ovfFilePath + " to retrive the number of disks present in OVA"); OVFHelper ovfHelper = new OVFHelper(); - List disks = ovfHelper.getOVFVolumeInfo(ovfFilePath); + List disks = ovfHelper.getOVFVolumeInfoFromFile(ovfFilePath, configurationId); return new GetDatadisksAnswer(disks); } catch (Exception e) { String msg = "Get Datadisk Template Count failed due to " + e.getMessage(); @@ -503,7 +505,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S throw new Exception(msg); } command = new Script("cp", _timeout, s_logger); - command.add(ovfFilePath + ".orig"); + command.add(ovfFilePath + ORIGINAL_FILE_EXTENSION); command.add(newTmplDirAbsolute); result = command.execute(); if (result != null) { @@ -517,7 +519,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S // Create OVF for the disk String newOvfFilePath = newTmplDirAbsolute + File.separator + ovfFilePath.substring(ovfFilePath.lastIndexOf(File.separator) + 1); OVFHelper ovfHelper = new OVFHelper(); - ovfHelper.rewriteOVFFile(ovfFilePath + ".orig", newOvfFilePath, diskName); + ovfHelper.rewriteOVFFileForSingleDisk(ovfFilePath + ORIGINAL_FILE_EXTENSION, newOvfFilePath, diskName); postCreatePrivateTemplate(newTmplDirAbsolute, templateId, templateUniqueName, physicalSize, virtualSize); writeMetaOvaForTemplate(newTmplDirAbsolute, ovfFilePath.substring(ovfFilePath.lastIndexOf(File.separator) + 1), diskName, templateUniqueName, physicalSize); diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java index c31bc9b38c6..fec8eab8ce2 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java @@ -37,6 +37,9 @@ import java.util.concurrent.Executors; import javax.naming.ConfigurationException; +import com.cloud.agent.api.storage.OVFEulaSectionTO; +import com.cloud.agent.api.storage.OVFVirtualHardwareSectionTO; +import com.cloud.agent.api.to.DatadiskTO; import com.cloud.agent.api.storage.OVFPropertyTO; import com.cloud.storage.template.Processor; import com.cloud.storage.template.S3TemplateDownloader; @@ -55,6 +58,7 @@ import com.cloud.storage.template.RawImageProcessor; import com.cloud.storage.template.TARProcessor; import com.cloud.storage.template.VhdProcessor; import com.cloud.storage.template.TemplateConstants; +import org.apache.cloudstack.api.net.NetworkPrerequisiteTO; import org.apache.cloudstack.storage.command.DownloadCommand; import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType; import org.apache.cloudstack.storage.command.DownloadProgressCommand; @@ -128,6 +132,10 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager private final long id; private final ResourceType resourceType; private List ovfProperties; + private List networks; + private List disks; + private OVFVirtualHardwareSectionTO hardwareSection; + private List eulaSections; public DownloadJob(TemplateDownloader td, String jobId, long id, String tmpltName, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum, String installPathPrefix, ResourceType resourceType) { @@ -230,6 +238,38 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager public void setOvfProperties(List ovfProperties) { this.ovfProperties = ovfProperties; } + + public List getNetworks() { + return networks; + } + + public void setNetworks(List networks) { + this.networks = networks; + } + + public List getDisks() { + return disks; + } + + public void setDisks(List disks) { + this.disks = disks; + } + + public void setVirtualHardwareSection(OVFVirtualHardwareSectionTO section) { + this.hardwareSection = section; + } + + public OVFVirtualHardwareSectionTO getVirtualHardwareSection() { + return this.hardwareSection; + } + + public List getEulaSections() { + return eulaSections; + } + + public void setEulaSections(List eulaSections) { + this.eulaSections = eulaSections; + } } public static final Logger LOGGER = Logger.getLogger(DownloadManagerImpl.class); @@ -509,7 +549,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager while (en.hasNext()) { Processor processor = en.next(); - FormatInfo info = null; + FormatInfo info; try { info = processor.process(resourcePath, null, templateName, this._processTimeout); } catch (InternalErrorException e) { @@ -526,6 +566,16 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager if (CollectionUtils.isNotEmpty(info.ovfProperties)) { dnld.setOvfProperties(info.ovfProperties); } + if (CollectionUtils.isNotEmpty(info.networks)) { + dnld.setNetworks(info.networks); + } + if (CollectionUtils.isNotEmpty(info.disks)) { + dnld.setDisks(info.disks); + } + dnld.setVirtualHardwareSection(info.hardwareSection); + if (CollectionUtils.isNotEmpty(info.eulaSections)) { + dnld.setEulaSections(info.eulaSections); + } break; } } @@ -829,6 +879,16 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager if (CollectionUtils.isNotEmpty(dj.getOvfProperties())) { answer.setOvfProperties(dj.getOvfProperties()); } + if (CollectionUtils.isNotEmpty(dj.getNetworks())) { + answer.setNetworkRequirements(dj.getNetworks()); + } + if (CollectionUtils.isNotEmpty(dj.getDisks())) { + answer.setDisks(dj.getDisks()); + } + answer.setOvfHardwareSection(dj.getVirtualHardwareSection()); + if (CollectionUtils.isNotEmpty(dj.getEulaSections())) { + answer.setEulaSections(dj.getEulaSections()); + } jobs.remove(jobId); return answer; default: diff --git a/test/integration/smoke/test_vm_life_cycle.py b/test/integration/smoke/test_vm_life_cycle.py index 68ec359f1a2..08668e47b72 100644 --- a/test/integration/smoke/test_vm_life_cycle.py +++ b/test/integration/smoke/test_vm_life_cycle.py @@ -27,7 +27,9 @@ from marvin.cloudstackAPI import (recoverVirtualMachine, migrateVirtualMachine, migrateVirtualMachineWithVolume, unmanageVirtualMachine, - listUnmanagedInstances) + listUnmanagedInstances, + listNics, + listVolumes) from marvin.lib.utils import * from marvin.lib.base import (Account, @@ -45,13 +47,17 @@ from marvin.lib.base import (Account, from marvin.lib.common import (get_domain, get_zone, get_suitable_test_template, + get_test_ovf_templates, list_hosts, - list_virtual_machines) + list_virtual_machines, + get_vm_vapp_configs) from marvin.codes import FAILED, PASS from nose.plugins.attrib import attr from marvin.lib.decoratorGenerators import skipTestIf # Import System modules import time +import json +from operator import itemgetter _multiprocess_shared_ = True @@ -1679,3 +1685,273 @@ class TestUnmanageVM(cloudstackTestCase): "PowerOn", "Unmanaged VM is still running" ) + + +class TestVAppsVM(cloudstackTestCase): + + @classmethod + def setUpClass(cls): + testClient = super(TestVAppsVM, cls).getClsTestClient() + cls.apiclient = testClient.getApiClient() + cls.services = testClient.getParsedTestDataConfig() + cls.hypervisor = testClient.getHypervisorInfo() + cls._cleanup = [] + + # Get Zone, Domain and templates + cls.domain = get_domain(cls.apiclient) + cls.zone = get_zone(cls.apiclient, cls.testClient.getZoneForTests()) + cls.services['mode'] = cls.zone.networktype + + cls.hypervisorNotSupported = cls.hypervisor.lower() != "vmware" + + if cls.hypervisorNotSupported == False: + + cls.account = Account.create( + cls.apiclient, + cls.services["account"], + domainid=cls.domain.id + ) + + cls.templates = get_test_ovf_templates( + cls.apiclient, + cls.zone.id, + cls.services['test_ovf_templates'], + cls.hypervisor + ) + if len(cls.templates) == 0: + assert False, "get_test_ovf_templates() failed to return templates" + + cls.custom_offering = ServiceOffering.create( + cls.apiclient, + cls.services["custom_service_offering"] + ) + + cls.isolated_network_offering = NetworkOffering.create( + cls.apiclient, + cls.services["isolated_network_offering"], + ) + cls.isolated_network_offering.update(cls.apiclient, state='Enabled') + + cls.l2_network_offering = NetworkOffering.create( + cls.apiclient, + cls.services["l2-network_offering"], + ) + cls.l2_network_offering.update(cls.apiclient, state='Enabled') + + cls._cleanup = [ + cls.account, + cls.custom_offering, + cls.isolated_network_offering, + cls.l2_network_offering + ] + + # Uncomment when tests are finished, to cleanup the test templates + for template in cls.templates: + cls._cleanup.append(template) + + @classmethod + def tearDownClass(cls): + try: + cleanup_resources(cls.apiclient, cls._cleanup) + except Exception as e: + raise Exception("Warning: Exception during class cleanup : %s" % e) + + def setUp(self): + self.apiclient = self.testClient.getApiClient() + self.cleanup = [] + + def tearDown(self): + try: + cleanup_resources(self.apiclient, self.cleanup) + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + + def get_ova_parsed_information_from_template(self, template): + if not template: + return None + details = template.details.__dict__ + configurations = [] + disks = [] + isos = [] + networks = [] + for propKey in details: + if propKey.startswith('ACS-configuration'): + configurations.append(json.loads(details[propKey])) + elif propKey.startswith('ACS-disk'): + detail = json.loads(details[propKey]) + if detail['isIso'] == True: + isos.append(detail) + else: + disks.append(detail) + elif propKey.startswith('ACS-network'): + networks.append(json.loads(details[propKey])) + + return configurations, disks, isos, networks + + def verify_nics(self, nic_networks, vm_id): + cmd = listNics.listNicsCmd() + cmd.virtualmachineid = vm_id + vm_nics = self.apiclient.listNics(cmd) + self.assertEqual( + isinstance(vm_nics, list), + True, + "Check listNics response returns a valid list" + ) + self.assertEqual( + len(nic_networks), + len(vm_nics), + msg="VM NIC count is different, expected = {}, result = {}".format(len(nic_networks), len(vm_nics)) + ) + nic_networks.sort(key=itemgetter('nic')) # CS will create NIC in order of InstanceID. Check network order + vm_nics.sort(key=itemgetter('deviceid')) + for i in range(len(vm_nics)): + nic = vm_nics[i] + nic_network = nic_networks[i] + self.assertEqual( + nic.networkid, + nic_network["network"], + msg="VM NIC(InstanceID: {}) network mismatch, expected = {}, result = {}".format(nic_network["nic"], nic_network["network"], nic.networkid) + ) + + def verify_disks(self, template_disks, vm_id): + cmd = listVolumes.listVolumesCmd() + cmd.virtualmachineid = vm_id + cmd.listall = True + vm_volumes = self.apiclient.listVolumes(cmd) + self.assertEqual( + isinstance(vm_volumes, list), + True, + "Check listVolumes response returns a valid list" + ) + self.assertEqual( + len(template_disks), + len(vm_volumes), + msg="VM volumes count is different, expected = {}, result = {}".format(len(template_disks), len(vm_volumes)) + ) + template_disks.sort(key=itemgetter('diskNumber')) + vm_volumes.sort(key=itemgetter('deviceid')) + for j in range(len(vm_volumes)): + volume = vm_volumes[j] + disk = template_disks[j] + self.assertEqual( + volume.size, + disk["virtualSize"], + msg="VM Volume(diskNumber: {}) network mismatch, expected = {}, result = {}".format(disk["diskNumber"], disk["virtualSize"], volume.size) + ) + + @attr(tags=["advanced", "advancedns", "smoke", "sg", "dev"], required_hardware="false") + @skipTestIf("hypervisorNotSupported") + def test_01_vapps_vm_cycle(self): + """ + Test the following for all found ovf templates: + 1. Deploy VM + 2. Verify VM has correct properties + 3. Verify VM has correct disks + 4. Verify VM has correct nics + 5. Destroy VM + """ + + for template in self.templates: + configurations, disks, isos, network = self.get_ova_parsed_information_from_template(template) + + if configurations: + conf = configurations[0] + items = conf['hardwareItems'] + cpu_speed = 1000 + cpu_number = 0 + memory = 0 + for item in items: + if item['resourceType'] == 'Memory': + memory = item['virtualQuantity'] + elif item['resourceType'] == 'Processor': + cpu_number = item['virtualQuantity'] + + nicnetworklist = [] + networks = [] + vm_service = self.services["virtual_machine_vapps"][template.name] + network_mappings = vm_service["nicnetworklist"] + for network_mapping in network_mappings: + network_service = self.services["isolated_network"] + network_offering_id = self.isolated_network_offering.id + if network_mapping["network"] == 'l2': + network_service = self.services["l2-network"] + network_offering_id = self.l2_network_offering.id + network = Network.create( + self.apiclient, + network_service, + networkofferingid=network_offering_id, + accountid=self.account.name, + domainid=self.account.domainid, + zoneid=self.zone.id) + networks.append(network) + for interface in network_mapping["nic"]: + nicnetworklist.append({"nic": interface, "network": network.id}) + + vm = VirtualMachine.create( + self.apiclient, + vm_service, + accountid=self.account.name, + domainid=self.account.domainid, + templateid=template.id, + serviceofferingid=self.custom_offering.id, + zoneid=self.zone.id, + customcpunumber=cpu_number, + customcpuspeed=cpu_speed, + custommemory=memory, + properties=vm_service['properties'], + nicnetworklist=nicnetworklist + ) + + list_vm_response = VirtualMachine.list( + self.apiclient, + id=vm.id + ) + self.debug( + "Verify listVirtualMachines response for virtual machine: %s" \ + % vm.id + ) + self.assertEqual( + isinstance(list_vm_response, list), + True, + "Check list response returns a valid list" + ) + self.assertNotEqual( + len(list_vm_response), + 0, + "Check VM available in List Virtual Machines" + ) + vm_response = list_vm_response[0] + self.assertEqual( + vm_response.id, + vm.id, + "Check virtual machine id in listVirtualMachines" + ) + self.assertEqual( + vm_response.name, + vm.name, + "Check virtual machine name in listVirtualMachines" + ) + self.assertEqual( + vm_response.state, + 'Running', + msg="VM is not in Running state" + ) + + # Verify nics + self.verify_nics(nicnetworklist, vm.id) + # Verify disks + self.verify_disks(disks, vm.id) + # Verify properties + original_properties = vm_service['properties'] + vm_properties = get_vm_vapp_configs(self.apiclient, self.config, self.zone, vm.instancename) + for property in original_properties: + if property["key"] in vm_properties: + self.assertEqual( + vm_properties[property["key"]], + property["value"], + "Check VM property %s with original value" % property["key"] + ) + + cmd = destroyVirtualMachine.destroyVirtualMachineCmd() + cmd.id = vm.id + self.apiclient.destroyVirtualMachine(cmd) diff --git a/tools/marvin/marvin/config/test_data.py b/tools/marvin/marvin/config/test_data.py index 47087176be2..436c656509d 100644 --- a/tools/marvin/marvin/config/test_data.py +++ b/tools/marvin/marvin/config/test_data.py @@ -986,7 +986,60 @@ test_data = { "ispublic": "True" } }, - + "test_ovf_templates": [ + { + "name": "test-ovf", + "displaytext": "test-ovf", + "format": "ova", + "hypervisor": "vmware", + "ostype": "Other Linux (64-bit)", + "url": "http://172.17.0.1/machina-2dd-iso.ova", + "deployasis": "True", + "requireshvm": "True", + "ispublic": "True" + } + ], + "virtual_machine_vapps": { + "test-ovf": { + "name": "testvm-vapps", + "displayname": "Test VM vApps", + "properties": [ + { + "key": "used.by.admin", + "value": "marvin" + }, + { + "key": "use.type", + "value": "test" + }, + { + "key": "usefull.property", + "value": "True" + } + ], + "nicnetworklist": [ + { + "network": "l2", + "nic": [15, 18] + }, + { + "network": "l2", + "nic": [16] + }, + { + "network": "l2", + "nic": [17] + } + ] + } + }, + "custom_service_offering": { + "name": "Custom Service Offering for vApps", + "displaytext": "Custom Service Offering for vApps", + "cpunumber": "", + "cpuspeed": "", + "memory": "" + }, "coreos_volume": { "diskname": "Volume_core", "urlvmware":"http://dl.openvm.eu/cloudstack/coreos/x86_64/coreos_production_cloudstack_image-vmware.ova", diff --git a/tools/marvin/marvin/lib/base.py b/tools/marvin/marvin/lib/base.py index 10ea666b2de..a53e295c974 100755 --- a/tools/marvin/marvin/lib/base.py +++ b/tools/marvin/marvin/lib/base.py @@ -502,7 +502,8 @@ class VirtualMachine: hostid=None, keypair=None, ipaddress=None, mode='default', method='GET', hypervisor=None, customcpunumber=None, customcpuspeed=None, custommemory=None, rootdisksize=None, - rootdiskcontroller=None, macaddress=None, datadisktemplate_diskoffering_list={}): + rootdiskcontroller=None, macaddress=None, datadisktemplate_diskoffering_list={}, + properties=None, nicnetworklist=None): """Create the instance""" cmd = deployVirtualMachine.deployVirtualMachineCmd() @@ -631,6 +632,12 @@ class VirtualMachine: elif macaddress in services: cmd.macaddress = services["macaddress"] + if properties: + cmd.properties = properties + + if nicnetworklist: + cmd.nicnetworklist = nicnetworklist + virtual_machine = apiclient.deployVirtualMachine(cmd, method=method) if 'password' in virtual_machine.__dict__.keys(): @@ -1414,6 +1421,9 @@ class Template: if "directdownload" in services: cmd.directdownload = services["directdownload"] + if "deployasis" in services: + cmd.deployasis = services["deployasis"] + # Register Template template = apiclient.registerTemplate(cmd) diff --git a/tools/marvin/marvin/lib/common.py b/tools/marvin/marvin/lib/common.py index 1c873a3d97a..fc9c72d88d6 100644 --- a/tools/marvin/marvin/lib/common.py +++ b/tools/marvin/marvin/lib/common.py @@ -395,6 +395,74 @@ def get_test_template(apiclient, zone_id=None, hypervisor=None, test_templates=N return FAILED +def get_test_ovf_templates(apiclient, zone_id=None, test_ovf_templates=None, hypervisor=None): + """ + @Name : get_test_ovf_templates + @Desc : Retrieves the list of test ovf templates used to running tests. When the template + is missing it will be download at most one in a zone for a hypervisor. + @Input : returns a list of templates + """ + result = [] + + if test_ovf_templates is None: + test_ovf_templates = test_data["test_ovf_templates"] + if test_ovf_templates is None: + return result + if hypervisor is None: + return result + hypervisor = hypervisor.lower() + if hypervisor != 'vmware': + return result + + for test_template in test_ovf_templates: + + cmd = listTemplates.listTemplatesCmd() + cmd.name = test_template['name'] + cmd.templatefilter = 'all' + if zone_id is not None: + cmd.zoneid = zone_id + if hypervisor is not None: + cmd.hypervisor = hypervisor + templates = apiclient.listTemplates(cmd) + + if validateList(templates)[0] != PASS: + template = Template.register(apiclient, test_template, zoneid=zone_id, hypervisor=hypervisor.lower(), randomize_name=False) + template.download(apiclient) + retries = 3 + while (template.details == None or len(template.details.__dict__) == 0) and retries > 0: + time.sleep(10) + template_list = Template.list(apiclient, id=template.id, zoneid=zone_id, templatefilter='all') + if isinstance(template_list, list): + template = Template(template_list[0].__dict__) + retries = retries - 1 + if template.details == None or len(template.details.__dict__) == 0: + template.delete(apiclient) + else: + result.append(template) + + if templates: + for template in templates: + if template.isready and template.ispublic and template.details != None and len(template.details.__dict__) > 0: + result.append(template.__dict__) + + return result + +def get_vm_vapp_configs(apiclient, config, setup_zone, vm_name): + zoneDetailsInConfig = [zone for zone in config.zones + if zone.name == setup_zone.name][0] + vcenterusername = zoneDetailsInConfig.vmwaredc.username + vcenterpassword = zoneDetailsInConfig.vmwaredc.password + vcenterip = zoneDetailsInConfig.vmwaredc.vcenter + vcenterObj = Vcenter( + vcenterip, + vcenterusername, + vcenterpassword) + + vms = vcenterObj.get_vms(vm_name) + if vms: + return vms[0]['vm']['properties'] + + def get_windows_template( apiclient, zone_id=None, ostype_desc=None, template_filter="featured", template_type='USER', template_id=None, template_name=None, account=None, domain_id=None, project_id=None, diff --git a/tools/marvin/marvin/lib/vcenter.py b/tools/marvin/marvin/lib/vcenter.py index d14f364bd1c..78ca50e19d8 100644 --- a/tools/marvin/marvin/lib/vcenter.py +++ b/tools/marvin/marvin/lib/vcenter.py @@ -114,6 +114,12 @@ class Vcenter(): for network in obj.network: vm_details['networks'].append({'name': network.name}) vm_details['raw'] = obj + vm_details['properties'] = {} + config = obj.config + if config and config.vAppConfig: + for prop in config.vAppConfig.property: + vm_details['properties'][prop.id] = prop.value + return vm_details def parse_details(self, obj, vimtype): diff --git a/ui/l10n/en.js b/ui/l10n/en.js index 8e25c54b04c..7e705131bcf 100644 --- a/ui/l10n/en.js +++ b/ui/l10n/en.js @@ -674,6 +674,7 @@ var dictionary = { "label.deleting.processing":"Deleting....", "label.deny":"Deny", "label.deployment.planner":"Deployment planner", +"label.deploy.as.is":"Deploy As-Is", "label.description":"Description", "label.destination.physical.network.id":"Destination physical network ID", "label.destination.zone":"Destination Zone", diff --git a/ui/scripts/docs.js b/ui/scripts/docs.js index a801a6b52e0..19a7ad0441b 100755 --- a/ui/scripts/docs.js +++ b/ui/scripts/docs.js @@ -1243,6 +1243,10 @@ cloudStack.docs = { desc: 'The Management Server will download the file from the specified URL, such as http://my.web.server/filename.vhd.gz', externalLink: '' }, + helpRegisterTemplateDeployAsIs: { + desc: 'Vmware Only: Deploy with specifications from OVF instead of orchestrated specs', + externalLink: '' + }, helpRegisterTemplateDirectDownload: { desc: 'KVM Only: Secondary Storage is bypassed and template/ISO is downloaded to Primary Storage on deployment', externalLink: '' diff --git a/ui/scripts/instanceWizard.js b/ui/scripts/instanceWizard.js index f06cd9046d4..466db99788b 100644 --- a/ui/scripts/instanceWizard.js +++ b/ui/scripts/instanceWizard.js @@ -965,17 +965,6 @@ } }); - if (selectedTemplateObj) { - $.ajax({ - url: createURL("listTemplateOvfProperties&id=" + selectedTemplateObj.id), - dataType: "json", - async: false, - success: function(json) { - ovfProps = json.listtemplateovfpropertiesresponse.ovfproperty; - } - }); - } - var $step = $('.step.sshkeyPairs:visible'); if (ovfProps == null || ovfProps.length === 0) { $step.addClass('next-skip-ovf-properties'); @@ -984,15 +973,6 @@ } }, - // Step PRE-8: Configure OVF Properties (if available) for the template - function(args) { - args.response.success({ - data: { - ovfProperties: ovfProps - } - }); - }, - // Step 8: Review function(args) { var $step = $('.step.review:visible'); @@ -1054,8 +1034,8 @@ } }); for (var k = 0; k < deployOvfProperties.length; k++) { - deployVmData["ovfproperties[" + k + "].key"] = deployOvfProperties[k].key; - deployVmData["ovfproperties[" + k + "].value"] = deployOvfProperties[k].value; + deployVmData["properties[" + k + "].key"] = deployOvfProperties[k].key; + deployVmData["properties[" + k + "].value"] = deployOvfProperties[k].value; } } diff --git a/ui/scripts/templates.js b/ui/scripts/templates.js index 1516286f01a..aeae6f04cc0 100644 --- a/ui/scripts/templates.js +++ b/ui/scripts/templates.js @@ -248,6 +248,7 @@ $form.find('.form-item[rel=rootDiskControllerType]').css('display', 'inline-block'); $form.find('.form-item[rel=nicAdapterType]').css('display', 'inline-block'); $form.find('.form-item[rel=keyboardType]').css('display', 'inline-block'); + $form.find('.form-item[rel=deployAsIs]').css('display', 'inline-block'); $form.find('.form-item[rel=xenserverToolsVersion61plus]').hide(); $form.find('.form-item[rel=rootDiskControllerTypeKVM]').hide(); $form.find('.form-item[rel=directdownload]').hide(); @@ -258,6 +259,7 @@ $form.find('.form-item[rel=keyboardType]').hide(); $form.find('.form-item[rel=rootDiskControllerTypeKVM]').hide(); $form.find('.form-item[rel=directdownload]').hide(); + $form.find('.form-item[rel=deployAsIs]').hide(); $form.find('.form-item[rel=requireshvm]').css('display', 'inline-block'); if (isAdmin()) { @@ -268,6 +270,7 @@ $form.find('.form-item[rel=nicAdapterType]').hide(); $form.find('.form-item[rel=keyboardType]').hide(); $form.find('.form-item[rel=xenserverToolsVersion61plus]').hide(); + $form.find('.form-item[rel=deployAsIs]').hide(); $form.find('.form-item[rel=rootDiskControllerTypeKVM]').css('display', 'inline-block'); $('#label_root_disk_controller').prop('selectedIndex', 2); $form.find('.form-item[rel=requireshvm]').css('display', 'inline-block'); @@ -281,6 +284,7 @@ $form.find('.form-item[rel=xenserverToolsVersion61plus]').hide(); $form.find('.form-item[rel=rootDiskControllerTypeKVM]').hide(); $form.find('.form-item[rel=directdownload]').hide(); + $form.find('.form-item[rel=deployAsIs]').hide(); $form.find('.form-item[rel=requireshvm]').css('display', 'inline-block'); } }); @@ -463,6 +467,13 @@ }); } }, + deployAsIs : { + label: 'label.deploy.as.is', + docID: 'helpRegisterTemplateDeployAsIs', + isBoolean: true, + dependsOn: 'hypervisor', + isHidden: true + }, // fields for hypervisor == "VMware" (ends here) format: { @@ -683,6 +694,11 @@ 'details[0].keyboard': args.data.keyboardType }); } + if (args.$form.find('.form-item[rel=deployAsIs]').css("display") != "none" && args.data.deployAsIs != "") { + $.extend(data, { + deployAsIs: (args.data.deployAsIs == "on") ? "true" : "false" + }); + } // for hypervisor == VMware (ends here) $.ajax({ @@ -1823,18 +1839,7 @@ } }, tabFilter: function (args) { - $.ajax({ - url: createURL("listTemplateOvfProperties&id=" + args.context.templates[0].id), - dataType: "json", - async: false, - success: function(json) { - ovfprops = json.listtemplateovfpropertiesresponse.ovfproperty; - } - }); var hiddenTabs = []; - if (ovfprops == null || ovfprops.length === 0) { - hiddenTabs.push("ovfpropertiestab"); - } return hiddenTabs; }, tabs: { @@ -1919,6 +1924,11 @@ isBoolean: true, converter: cloudStack.converters.toBooleanText }, + deployAsIs: { + label: 'label.deploy.as.is', + isBoolean: true, + converter: cloudStack.converters.toBooleanText + }, isextractable: { label: 'label.extractable.lower', isBoolean: true, @@ -2605,57 +2615,7 @@ } } }) - }, - - /** - * OVF properties tab (only displayed when OVF properties are available) - */ - ovfpropertiestab: { - title: 'label.ovf.properties', - listView: { - id: 'ovfproperties', - fields: { - label: { - label: 'label.label' - }, - description: { - label: 'label.description' - }, - value: { - label: 'label.value' - } - }, - hideSearchBar: true, - dataProvider: function(args) { - $.ajax({ - url: createURL("listTemplateOvfProperties"), - data: { - id: args.context.templates[0].id - }, - success: function(json) { - var ovfprops = json.listtemplateovfpropertiesresponse.ovfproperty; - var listDetails = []; - for (index in ovfprops){ - var prop = ovfprops[index]; - var det = {}; - det['label'] = prop['label']; - det['description'] = prop['description']; - det['value'] = prop['value']; - listDetails.push(det); - } - args.response.success({ - data: listDetails - }); - }, - - error: function(json) { - args.response.error(parseXMLHttpResponse(json)); - } - }); - - } - } - } + } } } } @@ -2851,6 +2811,11 @@ docID: 'helpRegisterISOFeatured', isBoolean: true, isHidden: true + }, + deployAsIs : { + label: 'label.deploy.as.is', + docID: 'helpRegisterTemplateDeployAsIs', + isBoolean: true } } }, @@ -2864,7 +2829,8 @@ zoneid: args.data.zone, isextractable: (args.data.isExtractable == "on"), bootable: (args.data.isBootable == "on"), - directdownload: (args.data.directdownload == "on") + directdownload: (args.data.directdownload == "on"), + deployAsIs: (args.data.deployAsIs == "on") }; if (args.$form.find('.form-item[rel=osTypeId]').css("display") != "none") { @@ -3701,6 +3667,11 @@ isBoolean: true, converter: cloudStack.converters.toBooleanText }, + deployAsIs: { + label: 'label.deploy.as.is', + isBoolean: true, + converter: cloudStack.converters.toBooleanText + }, size: { label: 'label.size', converter: function(args) { diff --git a/ui/scripts/ui-custom/instanceWizard.js b/ui/scripts/ui-custom/instanceWizard.js index 4aefa97b335..2004257b2c6 100644 --- a/ui/scripts/ui-custom/instanceWizard.js +++ b/ui/scripts/ui-custom/instanceWizard.js @@ -172,10 +172,20 @@ .html(qualifier) propertyField.append(option); }); - } else if (qualifiers.startsWith("MaxLen")) { - var length = qualifiers.replace("MaxLen(","").slice(0,-1); + } else if (qualifiers.includes("MinLen") || qualifiers.includes("MaxLen") ) { + var split = qualifiers.split(","); + var minLen = 0; + var maxLen = 524288; /* 524288 being the default according to w3schools */ + for ( var i = 0; i < split.length; i++ ) { + if (split[i].startsWith("MaxLen")) { + maxLen = split[i].replace("MaxLen(","").replace(")",""); + } else if (split[i].startsWith("MinLen")) { + minLen = split[i].replace("MinLen(","").replace(")",""); + } + } propertyField = $('') - .attr({maxlength : length, type: fieldType}) + .attr({pattern : '.{' + minLen + ',' + maxLen + '}'}) + .attr({type: fieldType}) .addClass('name').val(_s(this[fields.value])) } } else { diff --git a/utils/pom.xml b/utils/pom.xml index 8a745aa21a7..75c9cd463aa 100755 --- a/utils/pom.xml +++ b/utils/pom.xml @@ -178,6 +178,10 @@ jackson-databind ${cs.jackson.version} + + org.apache.commons + commons-compress + diff --git a/utils/src/main/java/com/cloud/utils/compression/CompressionUtil.java b/utils/src/main/java/com/cloud/utils/compression/CompressionUtil.java new file mode 100644 index 00000000000..20f0bc8eb7c --- /dev/null +++ b/utils/src/main/java/com/cloud/utils/compression/CompressionUtil.java @@ -0,0 +1,58 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package com.cloud.utils.compression; + +import org.apache.log4j.Logger; + +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; + +public class CompressionUtil { + + private static final Logger s_logger = Logger.getLogger(CompressionUtil.class); + + public byte[] compressString(String inputStr) throws IOException { + ByteArrayOutputStream obj = new ByteArrayOutputStream(); + GZIPOutputStream gzip = new GZIPOutputStream(obj); + gzip.write(inputStr.getBytes(StandardCharsets.UTF_8)); + gzip.close(); + return obj.toByteArray(); + } + + public String decompressByteArary(byte[] compressed) throws IOException { + GZIPInputStream gis = new GZIPInputStream(new ByteArrayInputStream(compressed)); + BufferedReader bf = new BufferedReader(new InputStreamReader(gis, StandardCharsets.UTF_8)); + String line = bf.readLine(); + StringBuilder builder = new StringBuilder(); + while (line != null) { + builder.append(line); + line = bf.readLine(); + if (line != null) { + builder.append("\n"); + } + } + return builder.toString(); + } +} diff --git a/utils/src/test/java/com/cloud/utils/compression/CompressionUtilTest.java b/utils/src/test/java/com/cloud/utils/compression/CompressionUtilTest.java new file mode 100644 index 00000000000..e247d6c9435 --- /dev/null +++ b/utils/src/test/java/com/cloud/utils/compression/CompressionUtilTest.java @@ -0,0 +1,128 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package com.cloud.utils.compression; + +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; + +public class CompressionUtilTest { + + private CompressionUtil util = new CompressionUtil(); + + private String testEula = "END USER LICENSE AGREEMENT\n" + + "\n" + + "IMPORTANT: PLEASE READ THIS END USER LICENSE AGREEMENT CAREFULLY. IT IS VERY IMPORTANT THAT YOU CHECK THAT YOU ARE PURCHASING CISCO SOFTWARE OR EQUIPMENT FROM AN APPROVED SOURCE AND THAT YOU, OR THE ENTITY YOU REPRESENT (COLLECTIVELY, THE \"CUSTOMER\") HAVE BEEN REGISTERED AS THE END USER FOR THE PURPOSES OF THIS CISCO END USER LICENSE AGREEMENT. IF YOU ARE NOT REGISTERED AS THE END USER YOU HAVE NO LICENSE TO USE THE SOFTWARE AND THE LIMITED WARRANTY IN THIS END USER LICENSE AGREEMENT DOES NOT APPLY. ASSUMING YOU HAVE PURCHASED FROM AN APPROVED SOURCE, DOWNLOADING, INSTALLING OR USING CISCO OR CISCO-SUPPLIED SOFTWARE CONSTITUTES ACCEPTANCE OF THIS AGREEMENT.\n" + + "\n" + + "CISCO SYSTEMS, INC. OR ITS AFFILIATE LICENSING THE SOFTWARE (\"CISCO\") IS WILLING TO LICENSE THIS SOFTWARE TO YOU ONLY UPON THE CONDITION THAT YOU PURCHASED THE SOFTWARE FROM AN APPROVED SOURCE AND THAT YOU ACCEPT ALL OF THE TERMS CONTAINED IN THIS END USER LICENSE AGREEMENT PLUS ANY ADDITIONAL LIMITATIONS ON THE LICENSE SET FORTH IN A SUPPLEMENTAL LICENSE AGREEMENT ACCOMPANYING THE PRODUCT, MADE AVAILABLE AT THE TIME OF YOUR ORDER, OR POSTED ON THE CISCO WEBSITE AT www.cisco.com/go/terms (COLLECTIVELY THE \"AGREEMENT\"). TO THE EXTENT OF ANY CONFLICT BETWEEN THE TERMS OF THIS END USER LICENSE AGREEMENT AND ANY SUPPLEMENTAL LICENSE AGREEMENT, THE SUPPLEMENTAL " + + "LICENSE AGREEMENT SHALL APPLY. BY DOWNLOADING, INSTALLING, OR USING THE SOFTWARE, YOU ARE REPRESENTING THAT YOU PURCHASED THE SOFTWARE FROM AN APPROVED SOURCE AND BINDING YOURSELF TO THE AGREEMENT. IF YOU DO NOT AGREE TO ALL OF THE TERMS OF THE AGREEMENT, THEN CISCO IS UNWILLING TO LICENSE THE SOFTWARE TO YOU AND (A) YOU MAY NOT DOWNLOAD, INSTALL OR USE THE SOFTWARE, AND (B) YOU MAY RETURN THE SOFTWARE (INCLUDING ANY UNOPENED CD PACKAGE AND ANY WRITTEN MATERIALS) FOR A FULL REFUND, OR, IF THE SOFTWARE AND WRITTEN MATERIALS ARE SUPPLIED AS PART OF ANOTHER PRODUCT, YOU MAY RETURN THE ENTIRE PRODUCT FOR A FULL REFUND. YOUR RIGHT TO RETURN AND REFUND EXPIRES 30 " + + "DAYS AFTER PURCHASE FROM AN APPROVED SOURCE, AND APPLIES ONLY IF YOU ARE THE ORIGINAL AND REGISTERED END USER PURCHASER. FOR THE PURPOSES OF THIS END USER LICENSE AGREEMENT, AN \"APPROVED SOURCE\" MEANS (A) CISCO; OR (B) A DISTRIBUTOR OR SYSTEMS INTEGRATOR AUTHORIZED BY CISCO TO DISTRIBUTE / SELL CISCO EQUIPMENT, SOFTWARE AND SERVICES WITHIN YOUR TERRITORY TO END USERS; OR (C) A RESELLER AUTHORIZED BY ANY SUCH DISTRIBUTOR OR SYSTEMS INTEGRATOR IN ACCORDANCE WITH THE TERMS OF THE DISTRIBUTOR'S AGREEMENT WITH CISCO TO DISTRIBUTE / SELL THE CISCO EQUIPMENT, SOFTWARE AND SERVICES WITHIN YOUR TERRITORY TO END USERS.\n" + + "\n" + + "THE FOLLOWING TERMS OF THE AGREEMENT GOVERN CUSTOMER'S USE OF THE SOFTWARE (DEFINED BELOW), EXCEPT TO THE EXTENT: (A) THERE IS A SEPARATE SIGNED CONTRACT BETWEEN CUSTOMER AND CISCO GOVERNING CUSTOMER'S USE OF THE SOFTWARE, OR (B) THE SOFTWARE INCLUDES A SEPARATE \"CLICK-ACCEPT\" LICENSE AGREEMENT OR THIRD PARTY LICENSE AGREEMENT AS PART OF THE INSTALLATION OR DOWNLOAD PROCESS GOVERNING CUSTOMER'S USE OF THE SOFTWARE. TO THE EXTENT OF A CONFLICT BETWEEN THE PROVISIONS OF THE FOREGOING DOCUMENTS, THE ORDER OF PRECEDENCE SHALL BE (1)THE SIGNED CONTRACT, (2) THE CLICK-ACCEPT AGREEMENT OR THIRD PARTY LICENSE AGREEMENT, AND (3) THE AGREEMENT. FOR PURPOSES OF THE " + + "AGREEMENT, \"SOFTWARE\" SHALL MEAN COMPUTER PROGRAMS, INCLUDING FIRMWARE AND COMPUTER PROGRAMS EMBEDDED IN CISCO EQUIPMENT, AS PROVIDED TO CUSTOMER BY AN APPROVED SOURCE, AND ANY UPGRADES, UPDATES, BUG FIXES OR MODIFIED VERSIONS THERETO (COLLECTIVELY, \"UPGRADES\"), ANY OF THE SAME WHICH HAS BEEN RELICENSED UNDER THE CISCO SOFTWARE TRANSFER AND RE-LICENSING POLICY (AS MAY BE AMENDED BY CISCO FROM TIME TO TIME) OR BACKUP COPIES OF ANY OF THE FOREGOING.\n" + + "\n" + + "License. Conditioned upon compliance with the terms and conditions of the Agreement, Cisco grants to Customer a nonexclusive and nontransferable license to use for Customer's internal business purposes the Software and the Documentation for which Customer has paid the required license fees to an Approved Source. \"Documentation\" means written information (whether contained in user or technical manuals, training materials, specifications or otherwise) pertaining to the Software and made available by an Approved Source with the Software in any manner (including on CD-Rom, or on-line). In order to use the Software, Customer may be required to input a registration number or product authorization key and register Customer's copy of the Software online at Cisco's website to obtain the necessary license key or license file.\n" + + "\n" + + "Customer's license to use the Software shall be limited to, and Customer shall not use the Software in excess of, a single hardware chassis or card or such other limitations as are set forth in the applicable Supplemental License Agreement or in the applicable purchase order which has been accepted by an Approved Source and for which Customer has paid to an Approved Source the required license fee (the \"Purchase Order\").\n" + + "\n" + + "Unless otherwise expressly provided in the Documentation or any applicable Supplemental License Agreement, Customer shall use the Software solely as embedded in, for execution on, or (where the applicable Documentation permits installation on non-Cisco equipment) for communication with Cisco equipment owned or leased by Customer and used for Customer's internal business purposes. No other licenses are granted by implication, estoppel or otherwise.\n" + + "\n" + + "For evaluation or beta copies for which Cisco does not charge a license fee, the above requirement to pay license fees does not apply.\n" + + "\n" + + "General Limitations. This is a license, not a transfer of title, to the Software and Documentation, and Cisco retains ownership of all copies of the Software and Documentation. Customer acknowledges that the Software and Documentation contain trade secrets of Cisco or its suppliers or licensors, including but not limited to the specific internal design and structure of individual programs and associated interface information. Except as otherwise expressly provided under the Agreement, Customer shall only use the Software in connection with the use of Cisco equipment purchased by the Customer from an Approved Source and Customer shall have no right, and Customer specifically agrees not to:\n" + + "\n" + + "(i) transfer, assign or sublicense its license rights to any other person or entity (other than in compliance with any Cisco relicensing/transfer policy then in force), or use the Software on Cisco equipment not purchased by the Customer from an Approved Source or on secondhand Cisco equipment, and Customer acknowledges that any attempted transfer, assignment, sublicense or use shall be void;\n" + + "\n" + + "(ii) make error corrections to or otherwise modify or adapt the Software or create derivative works based upon the Software, or permit third parties to do the same;\n" + + "\n" + + "(iii) reverse engineer or decompile, decrypt, disassemble or otherwise reduce the Software to human-readable form, except to the extent otherwise expressly permitted under applicable law notwithstanding this restriction or except to the extent that Cisco is legally required to permit such specific activity pursuant to any applicable open source license;\n" + + "\n" + + "(iv) publish any results of benchmark tests run on the Software;\n" + + "\n" + + "(v) use or permit the Software to be used to perform services for third parties, whether on a service bureau or time sharing basis or otherwise, without the express written authorization of Cisco; or\n" + + "\n" + + "(vi) disclose, provide, or otherwise make available trade secrets contained within the Software and Documentation in any form to any third party without the prior written consent of Cisco. Customer shall implement reasonable security measures to protect such trade secrets.\n" + + "\n" + + "To the extent required by applicable law, and at Customer's written request, Cisco shall provide Customer with the interface information needed to achieve interoperability between the Software and another independently created program, on payment of Cisco's applicable fee, if any. Customer shall observe strict obligations of confidentiality with respect to such information and shall use such information in compliance with any applicable terms and conditions upon which Cisco makes such information available.\n" + + "\n" + + "Software, Upgrades and Additional Copies. NOTWITHSTANDING ANY OTHER PROVISION OF THE AGREEMENT: (1) CUSTOMER HAS NO LICENSE OR RIGHT TO MAKE OR USE ANY ADDITIONAL COPIES OR UPGRADES UNLESS CUSTOMER, AT THE TIME OF MAKING OR ACQUIRING SUCH COPY OR UPGRADE, ALREADY HOLDS A VALID LICENSE TO THE ORIGINAL SOFTWARE AND HAS PAID THE APPLICABLE FEE TO AN APPROVED SOURCE FOR THE UPGRADE OR ADDITIONAL COPIES; (2) USE OF UPGRADES IS LIMITED TO CISCO EQUIPMENT SUPPLIED BY AN APPROVED SOURCE FOR WHICH CUSTOMER IS THE ORIGINAL END USER PURCHASER OR LESSEE OR OTHERWISE HOLDS A VALID LICENSE TO USE THE SOFTWARE WHICH IS BEING UPGRADED; AND (3) THE MAKING AND USE OF ADDITIONAL COPIES IS LIMITED TO NECESSARY BACKUP PURPOSES ONLY.\n" + + "\n" + + "Proprietary Notices. Customer agrees to maintain and reproduce all copyright, proprietary, and other notices on all copies, in any form, of the Software in the same form and manner that such copyright and other proprietary notices are included on the Software. Except as expressly authorized in the Agreement, Customer shall not make any copies or duplicates of any Software without the prior written permission of Cisco.\n" + + "\n" + + "Term and Termination. The Agreement and the license granted herein shall remain effective until terminated. Customer may terminate the Agreement and the license at any time by destroying all copies of Software and any Documentation. Customer's rights under the Agreement will terminate immediately without notice from Cisco if Customer fails to comply with any provision of the Agreement. Upon termination, Customer shall destroy all copies of Software and Documentation in its possession or control. All confidentiality obligations of Customer, " + + "all restrictions and limitations imposed on the Customer under the section titled \"General Limitations\" and all limitations of liability and disclaimers and restrictions of warranty shall survive termination of this Agreement. In addition, the provisions of the sections titled \"U.S. Government End User Purchasers\" and \"General Terms Applicable to the Limited Warranty Statement and End User License Agreement\" shall survive termination of the Agreement.\n" + + "\n" + + "Customer Records. Customer grants to Cisco and its independent accountants the right to examine Customer's books, records and accounts during Customer's normal business hours to verify compliance with this Agreement. In the event such audit discloses non-compliance with this Agreement, Customer shall promptly pay to Cisco the appropriate license fees, plus the reasonable cost of conducting the audit.\n" + + "\n" + + "Export, Re-Export, Transfer and Use Controls. The Software, Documentation and technology or direct products thereof (hereafter referred to as Software and Technology), supplied by Cisco under the Agreement are subject to export controls under the laws and regulations of the United States (\"U.S.\") and any other applicable countries' laws and regulations. Customer shall comply with such laws and regulations governing export, re-export, import, transfer and use of Cisco Software and Technology and will obtain all required U.S. and local authorizations, permits, or licenses. Cisco and Customer each agree to provide the other information, support documents, and assistance as may reasonably be required by the other in connection with securing authorizations or licenses. Information regarding compliance with export, re-export, transfer and use may be located at the following URL: www.cisco.com/web/about/doing_business/legal/global_export_trade/general_export/contract_compliance.html\n" + + "\n" + + "U.S. Government End User Purchasers. The Software and Documentation qualify as \"commercial items,\" as that term is defined at Federal Acquisition Regulation (\"FAR\") (48 C.F.R.) 2.101, consisting of \"commercial computer software\" and \"commercial computer software documentation\" as such terms are used in FAR 12.212. Consistent with FAR 12.212 and DoD FAR Supp. 227.7202-1 through 227.7202-4, and notwithstanding any other FAR or other contractual clause to the contrary in any agreement into which the Agreement may be incorporated, Customer may provide to Government end user or, if the Agreement is direct, Government end user will acquire, the Software and Documentation with only those rights set forth in the Agreement. Use of either the Software or Documentation or both constitutes agreement by the Government that the Software and Documentation are \"commercial computer software\" and \"commercial " + + "computer software documentation,\" and constitutes acceptance of the rights and restrictions herein.\n" + + "\n" + + "Identified Components; Additional Terms. The Software may contain or be delivered with one or more components, which may include third-party components, identified by Cisco in the Documentation, readme.txt file, third-party click-accept or elsewhere (e.g. on www.cisco.com) (the \"Identified Component(s)\") as being subject to different license agreement terms, disclaimers of warranties, limited warranties or other terms and conditions (collectively, \"Additional Terms\") than those set forth herein. You agree to the applicable Additional Terms for any such Identified Component(s).\n" + + "\n" + + "Limited Warranty\n" + + "\n" + + "Subject to the limitations and conditions set forth herein, Cisco warrants that commencing from the date of shipment to Customer (but in case of resale by an Approved Source other than Cisco, commencing not more than ninety (90) days after original shipment by Cisco), and continuing for a period of the longer of (a) ninety (90) days or (b) the warranty period (if any) expressly set forth as applicable specifically to software in the warranty card accompanying the product of which the Software is a part (the \"Product\") (if any): (a) the media on which the Software is furnished will be free of defects in materials and workmanship under normal use; and (b) the Software substantially conforms to the Documentation. The date of shipment of a Product by Cisco is set forth " + + "on the packaging material in which the Product is shipped. Except for the foregoing, the Software is provided \"AS IS\". This limited warranty extends only to the Software purchased from an Approved Source by a Customer who is the first registered end user. Customer's sole and exclusive remedy and the entire liability of Cisco and its suppliers under this limited warranty will be (i) replacement of defective media and/or (ii) at Cisco's option, repair, replacement, or refund of the purchase price of the Software, in both cases subject to the condition that any error or defect constituting a breach of this limited warranty is reported to the Approved Source supplying the Software to Customer, within the warranty period. Cisco or the Approved Source supplying the Software to Customer may, at its option, require return of the Software and/or Documentation as a condition to the remedy. " + + "In no event does Cisco warrant that the Software is error free or that Customer will be able to operate the Software without problems or interruptions. In addition, due to the continual development of new techniques for intruding upon and attacking networks, Cisco does not warrant that the Software or any equipment, system or network on which the Software is used will be free of vulnerability to intrusion or attack.\n" + + "\n" + + "Restrictions. This warranty does not apply if the Software, Product or any other equipment upon which the Software is authorized to be used (a) has been altered, except by Cisco or its authorized representative, (b) has not been installed, operated, repaired, or maintained in accordance with instructions supplied by Cisco, (c) has been subjected to abnormal physical or electrical stress, abnormal environmental conditions, misuse, negligence, or accident; or (d) is licensed for beta, evaluation, testing or demonstration purposes. The Software warranty also does not apply to (e) any temporary Software modules; (f) any Software not posted on Cisco's Software Center; (g) any Software that Cisco expressly provides on an \"AS IS\" basis on Cisco's Software Center; (h) any Software for which an Approved Source does not receive a license fee; and (i) Software supplied by any third party which is not an Approved Source.\n" + + "\n" + + "DISCLAIMER OF WARRANTY\n" + + "\n" + + "EXCEPT AS SPECIFIED IN THIS WARRANTY SECTION, ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS, AND WARRANTIES INCLUDING, WITHOUT LIMITATION, ANY IMPLIED WARRANTY OR CONDITION OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, SATISFACTORY QUALITY, NON-INTERFERENCE, ACCURACY OF INFORMATIONAL CONTENT, OR ARISING FROM A COURSE OF DEALING, LAW, USAGE, OR TRADE PRACTICE, ARE HEREBY EXCLUDED TO THE EXTENT ALLOWED BY APPLICABLE LAW AND ARE EXPRESSLY DISCLAIMED BY CISCO, ITS SUPPLIERS AND LICENSORS. TO THE EXTENT THAT ANY OF THE SAME CANNOT BE EXCLUDED, SUCH IMPLIED CONDITION, REPRESENTATION AND/OR WARRANTY IS LIMITED IN DURATION TO THE EXPRESS WARRANTY PERIOD REFERRED TO IN THE \"LIMITED WARRANTY\" SECTION ABOVE. BECAUSE SOME STATES OR JURISDICTIONS " + + "DO NOT ALLOW LIMITATIONS ON HOW LONG AN IMPLIED WARRANTY LASTS, THE ABOVE LIMITATION MAY NOT APPLY IN SUCH STATES. THIS WARRANTY GIVES CUSTOMER SPECIFIC LEGAL RIGHTS, AND CUSTOMER MAY ALSO HAVE OTHER RIGHTS WHICH VARY FROM JURISDICTION TO JURISDICTION. This disclaimer and exclusion shall apply even if the express warranty set forth above fails of its essential purpose.\n" + + "\n" + + "Disclaimer of Liabilities-Limitation of Liability. IF YOU ACQUIRED THE SOFTWARE IN THE UNITED STATES, LATIN AMERICA, CANADA, JAPAN OR THE CARIBBEAN, NOTWITHSTANDING ANYTHING ELSE IN THE AGREEMENT TO THE CONTRARY, ALL LIABILITY OF CISCO, ITS AFFILIATES, OFFICERS, DIRECTORS, EMPLOYEES, AGENTS, SUPPLIERS AND LICENSORS COLLECTIVELY, TO CUSTOMER, WHETHER IN CONTRACT, TORT (INCLUDING NEGLIGENCE), BREACH OF WARRANTY OR OTHERWISE, SHALL NOT EXCEED THE PRICE PAID BY CUSTOMER TO ANY APPROVED SOURCE FOR THE SOFTWARE THAT GAVE RISE TO THE CLAIM OR IF THE SOFTWARE IS PART OF ANOTHER PRODUCT, THE PRICE PAID FOR SUCH OTHER PRODUCT. THIS LIMITATION OF LIABILITY FOR SOFTWARE IS CUMULATIVE AND NOT PER INCIDENT (I.E. THE EXISTENCE OF TWO OR MORE CLAIMS WILL NOT ENLARGE THIS LIMIT).\n" + + "\n" + + "IF YOU ACQUIRED THE SOFTWARE IN EUROPE, THE MIDDLE EAST, AFRICA, ASIA OR OCEANIA, NOTWITHSTANDING ANYTHING ELSE IN THE AGREEMENT TO THE CONTRARY, ALL LIABILITY OF CISCO, ITS AFFILIATES, OFFICERS, DIRECTORS, EMPLOYEES, AGENTS, SUPPLIERS AND LICENSORS COLLECTIVELY, TO CUSTOMER, WHETHER IN CONTRACT, TORT (INCLUDING NEGLIGENCE), BREACH OF WARRANTY OR OTHERWISE, SHALL NOT EXCEED THE PRICE PAID BY CUSTOMER TO CISCO FOR THE SOFTWARE THAT GAVE RISE TO THE CLAIM OR IF THE SOFTWARE IS PART OF ANOTHER PRODUCT, THE PRICE PAID FOR SUCH OTHER PRODUCT. THIS LIMITATION OF LIABILITY FOR SOFTWARE IS CUMULATIVE AND NOT PER INCIDENT (I.E. THE EXISTENCE OF TWO OR MORE CLAIMS WILL NOT ENLARGE THIS LIMIT). NOTHING IN THE AGREEMENT SHALL LIMIT (I) THE LIABILITY OF CISCO, ITS AFFILIATES, OFFICERS, DIRECTORS, EMPLOYEES, AGENTS, SUPPLIERS AND LICENSORS TO CUSTOMER FOR PERSONAL " + + "INJURY OR DEATH CAUSED BY THEIR NEGLIGENCE, (II) CISCO'S LIABILITY FOR FRAUDULENT MISREPRESENTATION, OR (III) ANY LIABILITY OF CISCO WHICH CANNOT BE EXCLUDED UNDER APPLICABLE LAW.\n" + + "\n" + + "Disclaimer of Liabilities-Waiver of Consequential Damages and Other Losses. IF YOU ACQUIRED THE SOFTWARE IN THE UNITED STATES, LATIN AMERICA, THE CARIBBEAN OR CANADA, REGARDLESS OF WHETHER ANY REMEDY SET FORTH HEREIN FAILS OF ITS ESSENTIAL PURPOSE OR OTHERWISE, IN NO EVENT WILL CISCO OR ITS SUPPLIERS BE LIABLE FOR ANY LOST REVENUE, PROFIT, OR LOST OR DAMAGED DATA, BUSINESS INTERRUPTION, LOSS OF CAPITAL, OR FOR SPECIAL, INDIRECT, CONSEQUENTIAL, INCIDENTAL, OR PUNITIVE DAMAGES HOWEVER CAUSED AND REGARDLESS OF THE THEORY OF LIABILITY OR WHETHER ARISING OUT OF THE USE OF OR INABILITY TO USE SOFTWARE OR OTHERWISE AND EVEN IF CISCO OR ITS SUPPLIERS OR LICENSORS HAVE BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. BECAUSE SOME STATES OR JURISDICTIONS DO NOT ALLOW LIMITATION OR EXCLUSION OF CONSEQUENTIAL OR INCIDENTAL DAMAGES, THE ABOVE LIMITATION MAY NOT APPLY TO YOU.\n" + + "\n" + + "IF YOU ACQUIRED THE SOFTWARE IN JAPAN, EXCEPT FOR LIABILITY ARISING OUT OF OR IN CONNECTION WITH DEATH OR PERSONAL INJURY, FRAUDULENT MISREPRESENTATION, AND REGARDLESS OF WHETHER ANY REMEDY SET FORTH HEREIN FAILS OF ITS ESSENTIAL PURPOSE OR OTHERWISE, IN NO EVENT WILL CISCO, ITS AFFILIATES, OFFICERS, DIRECTORS, EMPLOYEES, AGENTS, SUPPLIERS AND LICENSORS BE LIABLE FOR ANY LOST REVENUE, PROFIT, OR LOST OR DAMAGED DATA, BUSINESS INTERRUPTION, LOSS OF CAPITAL, OR FOR SPECIAL, INDIRECT, CONSEQUENTIAL, INCIDENTAL, OR PUNITIVE DAMAGES HOWEVER CAUSED AND REGARDLESS OF THE THEORY OF LIABILITY OR WHETHER ARISING OUT OF THE USE OF OR INABILITY TO USE SOFTWARE OR OTHERWISE AND EVEN IF CISCO OR ANY APPROVED SOURCE OR THEIR SUPPLIERS OR LICENSORS HAVE BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.\n" + + "\n" + + "IF YOU ACQUIRED THE SOFTWARE IN EUROPE, THE MIDDLE EAST, AFRICA, ASIA OR OCEANIA, IN NO EVENT WILL CISCO, ITS AFFILIATES, OFFICERS, DIRECTORS, EMPLOYEES, AGENTS, SUPPLIERS AND LICENSORS, BE LIABLE FOR ANY LOST REVENUE, LOST PROFIT, OR LOST OR DAMAGED DATA, BUSINESS INTERRUPTION, LOSS OF CAPITAL, OR FOR SPECIAL, INDIRECT, CONSEQUENTIAL, INCIDENTAL, OR PUNITIVE DAMAGES, HOWSOEVER ARISING, INCLUDING, WITHOUT LIMITATION, IN CONTRACT, TORT (INCLUDING NEGLIGENCE) OR WHETHER ARISING OUT OF THE USE OF OR INABILITY TO USE THE SOFTWARE, EVEN IF, IN EACH CASE, CISCO, ITS AFFILIATES, OFFICERS, DIRECTORS, EMPLOYEES, AGENTS, SUPPLIERS AND LICENSORS, HAVE BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. BECAUSE SOME STATES OR JURISDICTIONS DO NOT ALLOW LIMITATION OR EXCLUSION OF CONSEQUENTIAL OR INCIDENTAL" + + " DAMAGES, THE ABOVE LIMITATION MAY NOT FULLY APPLY TO YOU. THE FOREGOING EXCLUSION SHALL NOT APPLY TO ANY LIABILITY ARISING OUT OF OR IN CONNECTION WITH: (I) DEATH OR PERSONAL INJURY, (II) FRAUDULENT MISREPRESENTATION, OR (III) CISCO'S LIABILITY IN CONNECTION WITH ANY TERMS THAT CANNOT BE EXCLUDED UNDER APPLICABLE LAW.\n" + + "\n" + + "Customer acknowledges and agrees that Cisco has set its prices and entered into the Agreement in reliance upon the disclaimers of warranty and the limitations of liability set forth herein, that the same reflect an allocation of risk between the parties (including the risk that a contract remedy may fail of its essential purpose and cause consequential loss), and that the same form an essential basis of the bargain between the parties.\n" + + "\n" + + "Controlling Law, Jurisdiction. If you acquired, by reference to the address on the purchase order accepted by the Approved Source, the Software in the United States, Latin America, or the Caribbean, the Agreement and warranties (\"Warranties\") are controlled by and construed under the laws of the State of California, United States of America, notwithstanding any conflicts of law provisions; and the state and federal courts of California shall have exclusive jurisdiction over any claim arising under the Agreement or Warranties. If you acquired the Software in Canada, unless expressly prohibited by local law, the Agreement and Warranties are controlled by and construed under the laws of the Province of Ontario, Canada, notwithstanding any conflicts of law provisions; and the courts of the " + + "Province of Ontario shall have exclusive jurisdiction over any claim arising under the Agreement or Warranties. If you acquired the Software in Europe, the Middle East, Africa, Asia or Oceania (excluding Australia), unless expressly prohibited by local law, the Agreement and Warranties are controlled by and construed under the laws of England, notwithstanding any conflicts of law provisions; and the English courts shall have exclusive jurisdiction over any " + + "claim arising under the Agreement or Warranties. In addition, if the Agreement is controlled by the laws of England, no person who is not a party to the Agreement shall be entitled to enforce or take the benefit of any of its terms under the Contracts (Rights of Third Parties) Act 1999. If you acquired the Software in Japan, unless expressly prohibited by local law, the Agreement and Warranties are controlled by and construed under the laws of Japan, notwithstanding any conflicts of law provisions; and the Tokyo District Court of Japan shall have exclusive jurisdiction over any claim arising under the Agreement or Warranties. If you acquired the Software in Australia, unless expressly prohibited by local law, the Agreement and Warranties are controlled by and construed under the laws of the " + + "State of New South Wales, Australia, notwithstanding any conflicts of law provisions; and the State and federal courts of New South Wales shall have exclusive jurisdiction over any claim arising under the Agreement or Warranties. If you acquired the Software in any other country, unless expressly prohibited by local law, the Agreement and Warranties are controlled by and construed under the laws of the State of California, United States of America, " + + "notwithstanding any conflicts of law provisions; and the state and federal courts of California shall have exclusive jurisdiction over any claim arising under the Agreement or Warranties.\n" + + "\n" + + "For all countries referred to above, the parties specifically disclaim the application of the UN Convention on Contracts for the International Sale of Goods. Notwithstanding the foregoing, either party may seek interim injunctive relief in any court of appropriate jurisdiction with respect to any alleged breach of such party's intellectual property or proprietary rights. If any portion hereof is found to be void or unenforceable, the remaining provisions of the Agreement and Warranties shall remain in full force and effect. Except as expressly provided herein, the Agreement constitutes the entire agreement between the parties with respect to the license of the Software and Documentation and supersedes any conflicting or additional terms contained in any Purchase Order or elsewhere, all of which terms are excluded. The Agreement has been written in the English language, and the parties agree that the English version will govern.\n" + + "\n" + + "Product warranty terms and other information applicable to Cisco products are available at the following URL: www.cisco.com/go/warranty\n" + + "\n" + + "Cisco and the Cisco logo are trademarks or registered trademarks of Cisco and/or its affiliates in the U.S. and other countries. To view a list of Cisco trademarks, go to this URL: www.cisco.com/go/trademarks. Third-party trademarks mentioned are the property of their respective owners. The use of the word partner does not imply a partnership relationship between Cisco and any other company. (1110R)\n" + + "\n" + + "© 1998, 2001, 2003, 2008-2014 Cisco Systems, Inc. All rights reserved."; + + @Test + public void testCompressDecompressString() throws IOException { + byte[] compressed = util.compressString(testEula); + String decompressed = util.decompressByteArary(compressed); + Assert.assertEquals(decompressed, testEula); + } +} diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/ClusterMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/ClusterMO.java index e4926472824..1bcebc9eab3 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/ClusterMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/ClusterMO.java @@ -386,7 +386,7 @@ public class ClusterMO extends BaseMO implements VmwareHypervisorHost { } @Override - public void importVmFromOVF(String ovfFilePath, String vmName, DatastoreMO dsMo, String diskOption) throws Exception { + public void importVmFromOVF(String ovfFilePath, String vmName, DatastoreMO dsMo, String diskOption, String configurationId) throws Exception { if (s_logger.isTraceEnabled()) s_logger.trace("vCenter API trace - importVmFromOVF(). target MOR: " + _mor.getValue() + ", ovfFilePath: " + ovfFilePath + ", vmName: " + vmName + ", datastore: " + dsMo.getMor().getValue() + ", diskOption: " + diskOption); @@ -397,7 +397,7 @@ public class ClusterMO extends BaseMO implements VmwareHypervisorHost { if (s_logger.isTraceEnabled()) s_logger.trace("vCenter API trace - importVmFromOVF(). resource pool: " + morRp.getValue()); - HypervisorHostHelper.importVmFromOVF(this, ovfFilePath, vmName, dsMo, diskOption, morRp, null); + HypervisorHostHelper.importVmFromOVF(this, ovfFilePath, vmName, dsMo, diskOption, morRp, null, configurationId); if (s_logger.isTraceEnabled()) s_logger.trace("vCenter API trace - importVmFromOVF() done"); diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HostMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HostMO.java index 057c4f8f769..0457039293a 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HostMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HostMO.java @@ -760,7 +760,7 @@ public class HostMO extends BaseMO implements VmwareHypervisorHost { return dsList; } - public void importVmFromOVF(String ovfFilePath, String vmName, String datastoreName, String diskOption) throws Exception { + public void importVmFromOVF(String ovfFilePath, String vmName, String datastoreName, String diskOption, String configurationId) throws Exception { if (s_logger.isTraceEnabled()) s_logger.trace("vCenter API trace - importVmFromOVF(). target MOR: " + _mor.getValue() + ", ovfFilePath: " + ovfFilePath + ", vmName: " + vmName + ",datastoreName: " + datastoreName + ", diskOption: " + diskOption); @@ -769,19 +769,19 @@ public class HostMO extends BaseMO implements VmwareHypervisorHost { if (dsMo == null) throw new Exception("Invalid datastore name: " + datastoreName); - importVmFromOVF(ovfFilePath, vmName, dsMo, diskOption); + importVmFromOVF(ovfFilePath, vmName, dsMo, diskOption, configurationId); if (s_logger.isTraceEnabled()) s_logger.trace("vCenter API trace - importVmFromOVF() done"); } @Override - public void importVmFromOVF(String ovfFilePath, String vmName, DatastoreMO dsMo, String diskOption) throws Exception { + public void importVmFromOVF(String ovfFilePath, String vmName, DatastoreMO dsMo, String diskOption, String configurationId) throws Exception { ManagedObjectReference morRp = getHyperHostOwnerResourcePool(); assert (morRp != null); - HypervisorHostHelper.importVmFromOVF(this, ovfFilePath, vmName, dsMo, diskOption, morRp, _mor); + HypervisorHostHelper.importVmFromOVF(this, ovfFilePath, vmName, dsMo, diskOption, morRp, _mor, configurationId); } @Override diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java index 3bb0d16b992..e8ea88207ac 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java @@ -37,6 +37,17 @@ import javax.xml.transform.TransformerFactory; import javax.xml.transform.dom.DOMSource; import javax.xml.transform.stream.StreamResult; +import com.vmware.vim25.ConcurrentAccessFaultMsg; +import com.vmware.vim25.DuplicateNameFaultMsg; +import com.vmware.vim25.FileFaultFaultMsg; +import com.vmware.vim25.InsufficientResourcesFaultFaultMsg; +import com.vmware.vim25.InvalidDatastoreFaultMsg; +import com.vmware.vim25.InvalidNameFaultMsg; +import com.vmware.vim25.InvalidStateFaultMsg; +import com.vmware.vim25.OutOfBoundsFaultMsg; +import com.vmware.vim25.RuntimeFaultFaultMsg; +import com.vmware.vim25.TaskInProgressFaultMsg; +import com.vmware.vim25.VmConfigFaultFaultMsg; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; @@ -1563,20 +1574,35 @@ public class HypervisorHostHelper { * - If the cluster hardware version is not set, check datacenter hardware version. If it is set, then it is set to vmConfig * - In case both cluster and datacenter hardware version are not set, hardware version is not set to vmConfig */ - protected static void setVMHardwareVersion(VirtualMachineConfigSpec vmConfig, ClusterMO clusterMO, DatacenterMO datacenterMO) throws Exception { + public static void setVMHardwareVersion(VirtualMachineConfigSpec vmConfig, ClusterMO clusterMO, DatacenterMO datacenterMO) throws Exception { + String version = getNewVMHardwareVersion(clusterMO, datacenterMO); + if (StringUtils.isNotBlank(version)) { + vmConfig.setVersion(version); + } + } + + /** + * Return the VM hardware version based on the information retrieved by the cluster and datacenter: + * - If the cluster hardware version is set, then return this hardware version + * - If the cluster hardware version is not set, check datacenter hardware version. If it is set, then return it + * - In case both cluster and datacenter hardware version are not set, return null + */ + public static String getNewVMHardwareVersion(ClusterMO clusterMO, DatacenterMO datacenterMO) throws Exception { + String version = null; ClusterConfigInfoEx clusterConfigInfo = clusterMO != null ? clusterMO.getClusterConfigInfo() : null; String clusterHardwareVersion = clusterConfigInfo != null ? clusterConfigInfo.getDefaultHardwareVersionKey() : null; if (StringUtils.isNotBlank(clusterHardwareVersion)) { s_logger.debug("Cluster hardware version found: " + clusterHardwareVersion + ". Creating VM with this hardware version"); - vmConfig.setVersion(clusterHardwareVersion); + version = clusterHardwareVersion; } else { DatacenterConfigInfo datacenterConfigInfo = datacenterMO != null ? datacenterMO.getDatacenterConfigInfo() : null; String datacenterHardwareVersion = datacenterConfigInfo != null ? datacenterConfigInfo.getDefaultHardwareVersionKey() : null; if (StringUtils.isNotBlank(datacenterHardwareVersion)) { s_logger.debug("Datacenter hardware version found: " + datacenterHardwareVersion + ". Creating VM with this hardware version"); - vmConfig.setVersion(datacenterHardwareVersion); + version = datacenterHardwareVersion; } } + return version; } private static VirtualDeviceConfigSpec getControllerSpec(String diskController, int busNum) { @@ -1714,6 +1740,11 @@ public class HypervisorHostHelper { return url; } + /** + * removes the NetworkSection element from the {ovfString} if it is an ovf xml file + * @param ovfString input string + * @return like the input string but if xml elements by name {NetworkSection} removed + */ public static String removeOVFNetwork(final String ovfString) { if (ovfString == null || ovfString.isEmpty()) { return ovfString; @@ -1748,8 +1779,12 @@ public class HypervisorHostHelper { return ovfString; } - public static void importVmFromOVF(VmwareHypervisorHost host, String ovfFilePath, String vmName, DatastoreMO dsMo, String diskOption, ManagedObjectReference morRp, - ManagedObjectReference morHost) throws Exception { + /** + * deploys a new VM from a ovf spec. It ignores network, defaults locale to 'US' + * @throws Exception shoud be a VmwareResourceException + */ + public static void importVmFromOVF(VmwareHypervisorHost host, String ovfFilePath, String vmName, DatastoreMO dsMo, String diskOption, ManagedObjectReference morRp, + ManagedObjectReference morHost, String configurationId) throws CloudRuntimeException, IOException { assert (morRp != null); @@ -1757,24 +1792,34 @@ public class HypervisorHostHelper { importSpecParams.setHostSystem(morHost); importSpecParams.setLocale("US"); importSpecParams.setEntityName(vmName); - importSpecParams.setDeploymentOption(""); + String deploymentOption = StringUtils.isNotBlank(configurationId) ? configurationId : ""; + importSpecParams.setDeploymentOption(deploymentOption); importSpecParams.setDiskProvisioning(diskOption); // diskOption: thin, thick, etc String ovfDescriptor = removeOVFNetwork(HttpNfcLeaseMO.readOvfContent(ovfFilePath)); VmwareContext context = host.getContext(); - OvfCreateImportSpecResult ovfImportResult = - context.getService().createImportSpec(context.getServiceContent().getOvfManager(), ovfDescriptor, morRp, dsMo.getMor(), importSpecParams); - + OvfCreateImportSpecResult ovfImportResult = null; + try { + ovfImportResult = context.getService().createImportSpec(context.getServiceContent().getOvfManager(), ovfDescriptor, morRp, dsMo.getMor(), importSpecParams); + } catch (ConcurrentAccessFaultMsg + | FileFaultFaultMsg + | InvalidDatastoreFaultMsg + | InvalidStateFaultMsg + | RuntimeFaultFaultMsg + | TaskInProgressFaultMsg + | VmConfigFaultFaultMsg error) { + throw new CloudRuntimeException("ImportSpec creation failed", error); + } if (ovfImportResult == null) { String msg = "createImportSpec() failed. ovfFilePath: " + ovfFilePath + ", vmName: " + vmName + ", diskOption: " + diskOption; s_logger.error(msg); - throw new Exception(msg); + throw new CloudRuntimeException(msg); } if(!ovfImportResult.getError().isEmpty()) { for (LocalizedMethodFault fault : ovfImportResult.getError()) { s_logger.error("createImportSpec error: " + fault.getLocalizedMessage()); } - throw new CloudException("Failed to create an import spec from " + ovfFilePath + ". Check log for details."); + throw new CloudRuntimeException("Failed to create an import spec from " + ovfFilePath + ". Check log for details."); } if (!ovfImportResult.getWarning().isEmpty()) { @@ -1783,22 +1828,55 @@ public class HypervisorHostHelper { } } - DatacenterMO dcMo = new DatacenterMO(context, host.getHyperHostDatacenter()); - ManagedObjectReference morLease = context.getService().importVApp(morRp, ovfImportResult.getImportSpec(), dcMo.getVmFolder(), morHost); + DatacenterMO dcMo = null; + try { + dcMo = new DatacenterMO(context, host.getHyperHostDatacenter()); + } catch (Exception e) { + throw new CloudRuntimeException(String.format("no datacenter for host '%s' available in context", context.getServerAddress()), e); + } + ManagedObjectReference folderMO = null; + try { + folderMO = dcMo.getVmFolder(); + } catch (Exception e) { + throw new CloudRuntimeException("no management handle for VmFolder", e); + } + ManagedObjectReference morLease = null; + try { + morLease = context.getService().importVApp(morRp, ovfImportResult.getImportSpec(), folderMO, morHost); + } catch (DuplicateNameFaultMsg + | FileFaultFaultMsg + | InsufficientResourcesFaultFaultMsg + | InvalidDatastoreFaultMsg + | InvalidNameFaultMsg + | OutOfBoundsFaultMsg + | RuntimeFaultFaultMsg + | VmConfigFaultFaultMsg fault) { + throw new CloudRuntimeException("import vApp failed",fault); + } if (morLease == null) { String msg = "importVApp() failed. ovfFilePath: " + ovfFilePath + ", vmName: " + vmName + ", diskOption: " + diskOption; s_logger.error(msg); - throw new Exception(msg); + throw new CloudRuntimeException(msg); } boolean importSuccess = true; final HttpNfcLeaseMO leaseMo = new HttpNfcLeaseMO(context, morLease); - HttpNfcLeaseState state = leaseMo.waitState(new HttpNfcLeaseState[] {HttpNfcLeaseState.READY, HttpNfcLeaseState.ERROR}); + HttpNfcLeaseState state = null; + try { + state = leaseMo.waitState(new HttpNfcLeaseState[] {HttpNfcLeaseState.READY, HttpNfcLeaseState.ERROR}); + } catch (Exception e) { + throw new CloudRuntimeException("exception while waiting for leaseMO", e); + } try { if (state == HttpNfcLeaseState.READY) { final long totalBytes = HttpNfcLeaseMO.calcTotalBytes(ovfImportResult); File ovfFile = new File(ovfFilePath); - HttpNfcLeaseInfo httpNfcLeaseInfo = leaseMo.getLeaseInfo(); + HttpNfcLeaseInfo httpNfcLeaseInfo = null; + try { + httpNfcLeaseInfo = leaseMo.getLeaseInfo(); + } catch (Exception e) { + throw new CloudRuntimeException("error waiting for lease info", e); + } List deviceUrls = httpNfcLeaseInfo.getDeviceUrl(); long bytesAlreadyWritten = 0; @@ -1809,6 +1887,7 @@ public class HypervisorHostHelper { for (OvfFileItem ovfFileItem : ovfImportResult.getFileItem()) { if (deviceKey.equals(ovfFileItem.getDeviceId())) { String absoluteFile = ovfFile.getParent() + File.separator + ovfFileItem.getPath(); + s_logger.info("Uploading file: " + absoluteFile); File f = new File(absoluteFile); if (f.exists()){ String urlToPost = deviceUrl.getUrl(); @@ -1828,31 +1907,44 @@ public class HypervisorHostHelper { String erroMsg = "File upload task failed to complete due to: " + e.getMessage(); s_logger.error(erroMsg); importSuccess = false; // Set flag to cleanup the stale template left due to failed import operation, if any - throw new Exception(erroMsg, e); + throw new CloudRuntimeException(erroMsg, e); } catch (Throwable th) { String errorMsg = "throwable caught during file upload task: " + th.getMessage(); s_logger.error(errorMsg); importSuccess = false; // Set flag to cleanup the stale template left due to failed import operation, if any - throw new Exception(errorMsg, th); + throw new CloudRuntimeException(errorMsg, th); } finally { progressReporter.close(); } if (bytesAlreadyWritten == totalBytes) { - leaseMo.updateLeaseProgress(100); + try { + leaseMo.updateLeaseProgress(100); + } catch (Exception e) { + throw new CloudRuntimeException("error while waiting for lease update", e); + } } } else if (state == HttpNfcLeaseState.ERROR) { - LocalizedMethodFault error = leaseMo.getLeaseError(); + LocalizedMethodFault error = null; + try { + error = leaseMo.getLeaseError(); + } catch (Exception e) { + throw new CloudRuntimeException("error getting lease error", e); + } MethodFault fault = error.getFault(); String erroMsg = "Object creation on vCenter failed due to: Exception: " + fault.getClass().getName() + ", message: " + error.getLocalizedMessage(); s_logger.error(erroMsg); - throw new Exception(erroMsg); + throw new CloudRuntimeException(erroMsg); } } finally { - if (!importSuccess) { - s_logger.error("Aborting the lease on " + vmName + " after import operation failed."); - leaseMo.abortLease(); - } else { - leaseMo.completeLease(); + try { + if (!importSuccess) { + s_logger.error("Aborting the lease on " + vmName + " after import operation failed."); + leaseMo.abortLease(); + } else { + leaseMo.completeLease(); + } + } catch (Exception e) { + throw new CloudRuntimeException("error completing lease", e); } } } diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java index 4650be3d73d..ed60d9157b7 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java @@ -35,6 +35,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import org.apache.commons.lang.StringUtils; @@ -3474,4 +3475,29 @@ public class VirtualMachineMO extends BaseMO { } return false; } + + /** + * Upgrades this virtual machine's virtual hardware to the latest revision that is supported by the virtual machine's current host. + * @param version If specified, upgrade to that specified version. If not specified, upgrade to the most current virtual hardware supported on the host. + * @return true if success, false if not + */ + public boolean upgradeVirtualHardwareVersion(String version) { + try { + String targetHwVersion = StringUtils.isNotBlank(version) ? version : "the highest available"; + s_logger.info("Upgrading the VM hardware version to " + targetHwVersion); + ManagedObjectReference morTask = _context.getService().upgradeVMTask(_mor, version); + boolean result = _context.getVimClient().waitForTask(morTask); + if (result) { + _context.waitForTaskProgressDone(morTask); + } else { + s_logger.error("VMware upgradeVMTask failed due to " + TaskMO.getTaskFailureInfo(_context, morTask)); + return false; + } + return true; + } catch (Exception e) { + String msg = "Attempted to upgrade VM hardware version failed: " + e.getMessage(); + s_logger.error(msg, e); + return false; + } + } } diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VmwareHypervisorHost.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VmwareHypervisorHost.java index bfa9db7a19b..ce2f178ef3f 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VmwareHypervisorHost.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VmwareHypervisorHost.java @@ -65,7 +65,7 @@ public interface VmwareHypervisorHost { int memoryReserveMB, String guestOsIdentifier, ManagedObjectReference morDs, boolean snapshotDirToParent, Pair controllerInfo, Boolean systemVm) throws Exception; - void importVmFromOVF(String ovfFilePath, String vmName, DatastoreMO dsMo, String diskOption) throws Exception; + void importVmFromOVF(String ovfFilePath, String vmName, DatastoreMO dsMo, String diskOption, String configurationId) throws Exception; ObjectContent[] getVmPropertiesOnHyperHost(String[] propertyPaths) throws Exception; diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareContext.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareContext.java index 807289f8bdf..ec11899ad3b 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareContext.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareContext.java @@ -367,32 +367,32 @@ public class VmwareContext { out.close(); } - public void uploadFile(String urlString, String localFileFullName) throws Exception { - uploadFile(urlString, new File(localFileFullName)); - } + public void uploadFile(String httpMethod, String urlString, String localFileName) throws Exception { + HttpURLConnection conn = getRawHTTPConnection(urlString); - public void uploadFile(String urlString, File localFile) throws Exception { - HttpURLConnection conn = getHTTPConnection(urlString, "PUT"); + conn.setDoOutput(true); + conn.setUseCaches(false); + + conn.setChunkedStreamingMode(ChunkSize); + conn.setRequestMethod(httpMethod); + conn.setRequestProperty("Connection", "Keep-Alive"); + String contentType = "application/octet-stream"; + conn.setRequestProperty("Content-Type", contentType); + conn.setRequestProperty("Content-Length", Long.toString(new File(localFileName).length())); + connectWithRetry(conn); OutputStream out = null; InputStream in = null; BufferedReader br = null; try { out = conn.getOutputStream(); - in = new FileInputStream(localFile); + in = new FileInputStream(localFileName); byte[] buf = new byte[ChunkSize]; int len = 0; while ((len = in.read(buf)) > 0) { out.write(buf, 0, len); } out.flush(); - - br = new BufferedReader(new InputStreamReader(conn.getInputStream(), getCharSetFromConnection(conn))); - String line; - while ((line = br.readLine()) != null) { - if (s_logger.isTraceEnabled()) - s_logger.trace("Upload " + urlString + " response: " + line); - } } finally { if (in != null) in.close(); @@ -402,6 +402,7 @@ public class VmwareContext { if (br != null) br.close(); + conn.disconnect(); } } @@ -427,7 +428,10 @@ public class VmwareContext { conn.setChunkedStreamingMode(ChunkSize); conn.setRequestMethod(httpMethod); conn.setRequestProperty("Connection", "Keep-Alive"); - conn.setRequestProperty("Content-Type", "application/x-vnd.vmware-streamVmdk"); + String contentType = urlString.endsWith(".iso") ? + "application/octet-stream" : + "application/x-vnd.vmware-streamVmdk"; + conn.setRequestProperty("Content-Type", contentType); conn.setRequestProperty("Content-Length", Long.toString(new File(localFileName).length())); connectWithRetry(conn); From cf2bb717920b56eb6e864e5b33b5cddbaf512623 Mon Sep 17 00:00:00 2001 From: nvazquez Date: Mon, 7 Sep 2020 14:59:52 -0300 Subject: [PATCH 077/164] Remove extra line after merging --- .../java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java | 1 - 1 file changed, 1 deletion(-) diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java index ed60d9157b7..f99c8f70393 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java @@ -37,7 +37,6 @@ import java.util.concurrent.Future; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; -import org.apache.commons.lang.StringUtils; import com.google.gson.Gson; import com.vmware.vim25.ArrayOfManagedObjectReference; From 5652097ef009dd19a1f6e97ce54bf2689616a308 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Mon, 31 Aug 2020 08:49:50 +0530 Subject: [PATCH 078/164] Added backup on fail over scenrio in case of new create disk method --- .../resource/VmwareStorageProcessor.java | 31 +++++++++---------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java index 1f1056a09aa..25fbdc4885d 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -2280,18 +2280,17 @@ public class VmwareStorageProcessor implements StorageProcessor { String volumeUuid = UUID.randomUUID().toString().replace("-", ""); String volumeDatastorePath = dsMo.getDatastorePath(volumeUuid + ".vmdk"); - - VirtualStorageObjectManagerMO vStorageObjectManagerMO = new VirtualStorageObjectManagerMO(context); - VStorageObject virtualDisk = vStorageObjectManagerMO.createDisk(morDatastore, VirtualDiskType.THIN, volume.getSize(), volumeDatastorePath, volumeUuid); VolumeObjectTO newVol = new VolumeObjectTO(); - DatastoreFile file = new DatastoreFile(((BaseConfigInfoDiskFileBackingInfo)virtualDisk.getConfig().getBacking()).getFilePath()); - newVol.setPath(file.getFileBaseName()); - newVol.setSize(volume.getSize()); - return new CreateObjectAnswer(newVol); - /* - * // This is old code which uses workervm to create disks - * String dummyVmName = hostService.getWorkerName(context, cmd, 0); + try { + VirtualStorageObjectManagerMO vStorageObjectManagerMO = new VirtualStorageObjectManagerMO(context); + VStorageObject virtualDisk = vStorageObjectManagerMO.createDisk(morDatastore, VirtualDiskType.THIN, volume.getSize(), volumeDatastorePath, volumeUuid); + DatastoreFile file = new DatastoreFile(((BaseConfigInfoDiskFileBackingInfo)virtualDisk.getConfig().getBacking()).getFilePath()); + newVol.setPath(file.getFileBaseName()); + newVol.setSize(volume.getSize()); + } catch (Exception e) { + s_logger.debug("Create disk using vStorageObject manager failed due to exception " + e.getMessage() + ", retying using worker VM"); + String dummyVmName = hostService.getWorkerName(context, cmd, 0); try { s_logger.info("Create worker VM " + dummyVmName); vmMo = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, dummyVmName); @@ -2304,14 +2303,14 @@ public class VmwareStorageProcessor implements StorageProcessor { vmMo.createDisk(volumeDatastorePath, (int)(volume.getSize() / (1024L * 1024L)), morDatastore, vmMo.getScsiDeviceControllerKey()); vmMo.detachDisk(volumeDatastorePath, false); } - catch (Exception e) { - s_logger.error("Deleting file " + volumeDatastorePath + " due to error: " + e.getMessage()); + catch (Exception e1) { + s_logger.error("Deleting file " + volumeDatastorePath + " due to error: " + e1.getMessage()); VmwareStorageLayoutHelper.deleteVolumeVmdkFiles(dsMo, volumeUuid, dcMo, VmwareManager.s_vmwareSearchExcludeFolder.value()); - throw new CloudRuntimeException("Unable to create volume due to: " + e.getMessage()); + throw new CloudRuntimeException("Unable to create volume due to: " + e1.getMessage()); } } - VolumeObjectTO newVol = new VolumeObjectTO(); + newVol = new VolumeObjectTO(); newVol.setPath(volumeUuid); newVol.setSize(volume.getSize()); return new CreateObjectAnswer(newVol); @@ -2322,8 +2321,8 @@ public class VmwareStorageProcessor implements StorageProcessor { vmMo.destroy(); } } - - * */ + } + return new CreateObjectAnswer(newVol); } catch (Throwable e) { if (e instanceof RemoteException) { s_logger.warn("Encounter remote exception to vCenter, invalidate VMware session context"); From 700ab1a5ef5716861b9721cf4e7a7e0a5e7f51a4 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Wed, 9 Sep 2020 07:57:08 +0530 Subject: [PATCH 079/164] vVols datastore support added --- .../vmware/manager/VmwareHostService.java | 3 +- .../manager/VmwareStorageManagerImpl.java | 10 +- .../vmware/resource/VmwareResource.java | 9 +- ...VmwareSecondaryStorageResourceHandler.java | 3 +- .../resource/VmwareStorageProcessor.java | 176 ++++++++++++------ .../vmware/mo/HypervisorHostHelper.java | 3 + 6 files changed, 135 insertions(+), 69 deletions(-) diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareHostService.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareHostService.java index 14630b3b89e..ea97a6e8d54 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareHostService.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareHostService.java @@ -17,6 +17,7 @@ package com.cloud.hypervisor.vmware.manager; import com.cloud.agent.api.Command; +import com.cloud.hypervisor.vmware.mo.DatastoreMO; import com.cloud.hypervisor.vmware.mo.VmwareHypervisorHost; import com.cloud.hypervisor.vmware.util.VmwareContext; @@ -27,5 +28,5 @@ public interface VmwareHostService { VmwareHypervisorHost getHyperHost(VmwareContext context, Command cmd); - String getWorkerName(VmwareContext context, Command cmd, int workerSequence); + String getWorkerName(VmwareContext context, Command cmd, int workerSequence, DatastoreMO dsMo) throws Exception; } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareStorageManagerImpl.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareStorageManagerImpl.java index 2fcf1358d08..3e2b5a02f6c 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareStorageManagerImpl.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareStorageManagerImpl.java @@ -343,7 +343,7 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { if (vmMo == null) { dsMo = new DatastoreMO(hyperHost.getContext(), morDs); - workerVMName = hostService.getWorkerName(context, cmd, 0); + workerVMName = hostService.getWorkerName(context, cmd, 0, dsMo); vmMo = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, workerVMName, null); if (vmMo == null) { @@ -362,7 +362,7 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { } snapshotBackupUuid = backupSnapshotToSecondaryStorage(vmMo, accountId, volumeId, cmd.getVolumePath(), snapshotUuid, secondaryStorageUrl, prevSnapshotUuid, - prevBackupUuid, hostService.getWorkerName(context, cmd, 1), cmd.getNfsVersion()); + prevBackupUuid, hostService.getWorkerName(context, cmd, 1, dsMo), cmd.getNfsVersion()); success = (snapshotBackupUuid != null); if (success) { @@ -428,7 +428,7 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { } Ternary result = createTemplateFromVolume(vmMo, accountId, templateId, cmd.getUniqueName(), secondaryStoragePoolURL, volumePath, - hostService.getWorkerName(context, cmd, 0), cmd.getNfsVersion()); + hostService.getWorkerName(context, cmd, 0, null), cmd.getNfsVersion()); return new CreatePrivateTemplateAnswer(cmd, true, null, result.first(), result.third(), result.second(), cmd.getUniqueName(), ImageFormat.OVA); @@ -486,7 +486,7 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { Pair result; if (cmd.toSecondaryStorage()) { result = copyVolumeToSecStorage(hostService, hyperHost, cmd, vmName, volumeId, cmd.getPool().getUuid(), volumePath, secondaryStorageURL, - hostService.getWorkerName(context, cmd, 0), cmd.getNfsVersion()); + hostService.getWorkerName(context, cmd, 0, null), cmd.getNfsVersion()); } else { StorageFilerTO poolTO = cmd.getPool(); @@ -1025,7 +1025,7 @@ public class VmwareStorageManagerImpl implements VmwareStorageManager { vmMo.createSnapshot(exportName, "Temporary snapshot for copy-volume command", false, false); } - exportVolumeToSecondaryStorage(vmMo, volumePath, secStorageUrl, "volumes/" + volumeFolder, exportName, hostService.getWorkerName(hyperHost.getContext(), cmd, 1), + exportVolumeToSecondaryStorage(vmMo, volumePath, secStorageUrl, "volumes/" + volumeFolder, exportName, hostService.getWorkerName(hyperHost.getContext(), cmd, 1, null), nfsVersion, clonedWorkerVMNeeded); return new Pair(volumeFolder, exportName); diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 5d611f4be67..54c3b4449af 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -815,7 +815,6 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa // OfflineVmwareMigration: this method is 100 lines and needs refactorring anyway // we need to spawn a worker VM to attach the volume to and resize the volume. useWorkerVm = true; - vmName = getWorkerName(getServiceContext(), cmd, 0); String poolId = cmd.getPoolUuid(); @@ -823,6 +822,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa // OfflineVmwareMigration: 1. find data(store) ManagedObjectReference morDS = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, poolId); DatastoreMO dsMo = new DatastoreMO(hyperHost.getContext(), morDS); + vmName = getWorkerName(getServiceContext(), cmd, 0, dsMo); s_logger.info("Create worker VM " + vmName); @@ -4768,7 +4768,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa // OfflineVmwareMigration: we need to refactor the worker vm creation out for use in migration methods as well as here // OfflineVmwareMigration: this method is 100 lines and needs refactorring anyway // we need to spawn a worker VM to attach the volume to and move it - vmName = getWorkerName(getServiceContext(), cmd, 0); + vmName = getWorkerName(getServiceContext(), cmd, 0, dsMo); // OfflineVmwareMigration: refactor for re-use // OfflineVmwareMigration: 1. find data(store) @@ -6967,9 +6967,12 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa @Override @DB - public String getWorkerName(VmwareContext context, Command cmd, int workerSequence) { + public String getWorkerName(VmwareContext context, Command cmd, int workerSequence, DatastoreMO dsMo) throws Exception { VmwareManager mgr = context.getStockObject(VmwareManager.CONTEXT_STOCK_NAME); String vmName = mgr.composeWorkerName(); + if (dsMo!= null && dsMo.getDatastoreType().equalsIgnoreCase("VVOL")) { + vmName = CustomFieldConstants.CLOUD_UUID + "-" + vmName; + } assert (cmd != null); context.getStockObject(VmwareManager.CONTEXT_STOCK_NAME); diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareSecondaryStorageResourceHandler.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareSecondaryStorageResourceHandler.java index b58253d6650..936ca15a4a5 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareSecondaryStorageResourceHandler.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareSecondaryStorageResourceHandler.java @@ -41,6 +41,7 @@ import com.cloud.hypervisor.vmware.manager.VmwareStorageManager; import com.cloud.hypervisor.vmware.manager.VmwareStorageManagerImpl; import com.cloud.hypervisor.vmware.manager.VmwareStorageMount; import com.cloud.hypervisor.vmware.mo.ClusterMO; +import com.cloud.hypervisor.vmware.mo.DatastoreMO; import com.cloud.hypervisor.vmware.mo.HostMO; import com.cloud.hypervisor.vmware.mo.VmwareHostType; import com.cloud.hypervisor.vmware.mo.VmwareHypervisorHost; @@ -297,7 +298,7 @@ public class VmwareSecondaryStorageResourceHandler implements SecondaryStorageRe } @Override - public String getWorkerName(VmwareContext context, Command cmd, int workerSequence) { + public String getWorkerName(VmwareContext context, Command cmd, int workerSequence, DatastoreMO dsMo) { assert (cmd.getContextParam("worker") != null); assert (workerSequence < 2); diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java index 25fbdc4885d..5ff4ee33808 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -399,7 +399,7 @@ public class VmwareStorageProcessor implements StorageProcessor { // If vmName is not null, then move all VMDK files out of this folder to the root folder and then delete the folder named vmName. if (vmName != null) { - String workerVmName = hostService.getWorkerName(context, cmd, 0); + String workerVmName = hostService.getWorkerName(context, cmd, 0, dsMo); VirtualMachineMO vmMo = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, workerVmName, null); @@ -484,9 +484,9 @@ public class VmwareStorageProcessor implements StorageProcessor { private Pair copyTemplateFromSecondaryToPrimary(VmwareHypervisorHost hyperHost, DatastoreMO datastoreMo, String secondaryStorageUrl, String templatePathAtSecondaryStorage, String templateName, String templateUuid, - boolean createSnapshot, String nfsVersion) throws Exception { + boolean createSnapshot, String nfsVersion, String configuration) throws Exception { s_logger.info("Executing copyTemplateFromSecondaryToPrimary. secondaryStorage: " + secondaryStorageUrl + ", templatePathAtSecondaryStorage: " + - templatePathAtSecondaryStorage + ", templateName: " + templateName); + templatePathAtSecondaryStorage + ", templateName: " + templateName + ", configuration: " + configuration); String secondaryMountPoint = mountService.getMountPoint(secondaryStorageUrl, nfsVersion); s_logger.info("Secondary storage mount point: " + secondaryMountPoint); @@ -517,11 +517,16 @@ public class VmwareStorageProcessor implements StorageProcessor { throw new Exception(msg); } - String vmName = templateUuid; - hyperHost.importVmFromOVF(srcFileName, vmName, datastoreMo, "thin"); + if (datastoreMo.getDatastoreType().equalsIgnoreCase("VVOL")) { + templateUuid = CustomFieldConstants.CLOUD_UUID + "-" + templateUuid; + } - VirtualMachineMO vmMo = hyperHost.findVmOnHyperHost(vmName); VmConfigInfo vAppConfig; + if (s_logger.isDebugEnabled()) { + s_logger.debug(String.format("Deploying OVF template %s with configuration %s", templateName, configuration)); + } + hyperHost.importVmFromOVF(srcFileName, templateUuid, datastoreMo, "thin", configuration); + VirtualMachineMO vmMo = hyperHost.findVmOnHyperHost(templateUuid); if (vmMo == null) { String msg = "Failed to import OVA template. secondaryStorage: " + secondaryStorageUrl + ", templatePathAtSecondaryStorage: " + templatePathAtSecondaryStorage + @@ -577,6 +582,7 @@ public class VmwareStorageProcessor implements StorageProcessor { DataTO destData = cmd.getDestTO(); DataStoreTO destStore = destData.getDataStore(); DataStoreTO primaryStore = destStore; + String configurationId = ((TemplateObjectTO) destData).getDeployAsIsConfiguration(); String secondaryStorageUrl = nfsImageStore.getUrl(); @@ -634,7 +640,13 @@ public class VmwareStorageProcessor implements StorageProcessor { try { String storageUuid = managed ? managedStoragePoolName : primaryStore.getUuid(); - String templateUuidName = deriveTemplateUuidOnHost(hyperHost, storageUuid, templateInfo.second()); + + // Generate a new template uuid if the template is marked as deploy-as-is, + // as it supports multiple configurations + String templateUuidName = template.isDeployAsIs() ? + UUID.randomUUID().toString() : + deriveTemplateUuidOnHost(hyperHost, storageUuid, templateInfo.second()); + DatacenterMO dcMo = new DatacenterMO(context, hyperHost.getHyperHostDatacenter()); VirtualMachineMO templateMo = VmwareHelper.pickOneVmOnRunningHost(dcMo.findVmByNameAndLabel(templateUuidName), true); Pair vmInfo = null; @@ -660,7 +672,7 @@ public class VmwareStorageProcessor implements StorageProcessor { if (managed) { vmInfo = copyTemplateFromSecondaryToPrimary(hyperHost, dsMo, secondaryStorageUrl, templateInfo.first(), templateInfo.second(), - managedStoragePoolRootVolumeName, false, _nfsVersion); + managedStoragePoolRootVolumeName, false, _nfsVersion, configurationId); VirtualMachineMO vmMo = vmInfo.first(); vmMo.unregisterVm(); @@ -680,7 +692,7 @@ public class VmwareStorageProcessor implements StorageProcessor { } else { vmInfo = copyTemplateFromSecondaryToPrimary(hyperHost, dsMo, secondaryStorageUrl, templateInfo.first(), templateInfo.second(), - templateUuidName, true, _nfsVersion); + templateUuidName, true, _nfsVersion, configurationId); } } else { s_logger.info("Template " + templateInfo.second() + " has already been setup, skip the template setup process in primary storage"); @@ -696,9 +708,14 @@ public class VmwareStorageProcessor implements StorageProcessor { } } else { - newTemplate.setPath(templateUuidName); + if (dsMo.getDatastoreType().equalsIgnoreCase("VVOL")) { + newTemplate.setPath(CustomFieldConstants.CLOUD_UUID + "-" + templateUuidName); + } else { + newTemplate.setPath(templateUuidName); + } } + newTemplate.setDeployAsIsConfiguration(configurationId); newTemplate.setSize((vmInfo != null)? vmInfo.second() : new Long(0)); return new CopyCmdAnswer(newTemplate); @@ -801,7 +818,7 @@ public class VmwareStorageProcessor implements StorageProcessor { } else { if (srcStore == null) { // create a root volume for blank VM (created from ISO) - String dummyVmName = hostService.getWorkerName(context, cmd, 0); + String dummyVmName = hostService.getWorkerName(context, cmd, 0, dsMo); try { vmMo = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, dummyVmName, null); @@ -834,42 +851,10 @@ public class VmwareStorageProcessor implements StorageProcessor { s_logger.warn("Template host in vSphere is not in connected state, request template reload"); return new CopyCmdAnswer("Template host in vSphere is not in connected state, request template reload"); } - - ManagedObjectReference morPool = hyperHost.getHyperHostOwnerResourcePool(); - ManagedObjectReference morCluster = hyperHost.getHyperHostCluster(); - if (template.getSize() != null){ - _fullCloneFlag = volume.getSize() > template.getSize() ? true : _fullCloneFlag; - } - if (!_fullCloneFlag) { - createVMLinkedClone(vmTemplate, dcMo, vmdkName, morDatastore, morPool); + if (dsMo.getDatastoreType().equalsIgnoreCase("VVOL") && volume.getVolumeType() == Volume.Type.ROOT) { + vmdkFileBaseName = cloneVMwithVMname(context, hyperHost, template, vmTemplate, volume, dcMo, dsMo); } else { - createVMFullClone(vmTemplate, dcMo, dsMo, vmdkName, morDatastore, morPool); - } - - vmMo = new ClusterMO(context, morCluster).findVmOnHyperHost(vmdkName); - assert (vmMo != null); - - vmdkFileBaseName = vmMo.getVmdkFileBaseNames().get(0); - s_logger.info("Move volume out of volume-wrapper VM " + vmdkFileBaseName); - String[] vmwareLayoutFilePair = VmwareStorageLayoutHelper.getVmdkFilePairDatastorePath(dsMo, vmdkName, vmdkFileBaseName, VmwareStorageLayoutType.VMWARE, !_fullCloneFlag); - String[] legacyCloudStackLayoutFilePair = VmwareStorageLayoutHelper.getVmdkFilePairDatastorePath(dsMo, vmdkName, vmdkFileBaseName, VmwareStorageLayoutType.CLOUDSTACK_LEGACY, !_fullCloneFlag); - - for (int i=0; i template.getSize() ? true : _fullCloneFlag; + } + if (!_fullCloneFlag) { + createVMLinkedClone(vmTemplate, dcMo, volume.getVmName(), morDatastore, morPool); + } else { + createVMFullClone(vmTemplate, dcMo, dsMo, volume.getVmName(), morDatastore, morPool); + } + + VirtualMachineMO vmMo = new ClusterMO(context, morCluster).findVmOnHyperHost(volume.getVmName()); + assert (vmMo != null); + + return vmMo.getVmdkFileBaseNames().get(0); + } + + private String createVMFolderWithVMName(VmwareContext context, VmwareHypervisorHost hyperHost, TemplateObjectTO template, + VirtualMachineMO vmTemplate, VolumeObjectTO volume, DatacenterMO dcMo, DatastoreMO dsMo, + String searchExcludedFolders) throws Exception { + ManagedObjectReference morDatastore = dsMo.getMor(); + ManagedObjectReference morPool = hyperHost.getHyperHostOwnerResourcePool(); + ManagedObjectReference morCluster = hyperHost.getHyperHostCluster(); + String vmdkName = volume.getName(); + if (template.getSize() != null){ + _fullCloneFlag = volume.getSize() > template.getSize() ? true : _fullCloneFlag; + } + if (!_fullCloneFlag) { + createVMLinkedClone(vmTemplate, dcMo, vmdkName, morDatastore, morPool); + } else { + createVMFullClone(vmTemplate, dcMo, dsMo,vmdkName, morDatastore, morPool); + } + + VirtualMachineMO vmMo = new ClusterMO(context, morCluster).findVmOnHyperHost(vmdkName); + assert (vmMo != null); + + String vmdkFileBaseName = vmMo.getVmdkFileBaseNames().get(0); + s_logger.info("Move volume out of volume-wrapper VM " + vmdkFileBaseName); + String[] vmwareLayoutFilePair = VmwareStorageLayoutHelper.getVmdkFilePairDatastorePath(dsMo, vmdkName, vmdkFileBaseName, VmwareStorageLayoutType.VMWARE, !_fullCloneFlag); + String[] legacyCloudStackLayoutFilePair = VmwareStorageLayoutHelper.getVmdkFilePairDatastorePath(dsMo, vmdkName, vmdkFileBaseName, VmwareStorageLayoutType.CLOUDSTACK_LEGACY, !_fullCloneFlag); + + for (int i=0; i(destVolumePath, exportName); } finally { @@ -1066,7 +1125,7 @@ public class VmwareStorageProcessor implements StorageProcessor { result = copyVolumeToSecStorage(hostService, hyperHost, cmd, vmName, primaryStorage.getUuid(), srcVolume.getPath(), destVolume.getPath(), destStore.getUrl(), - hostService.getWorkerName(context, cmd, 0)); + hostService.getWorkerName(context, cmd, 0, null)); VolumeObjectTO newVolume = new VolumeObjectTO(); newVolume.setPath(result.first() + File.separator + result.second()); return new CopyCmdAnswer(newVolume); @@ -1223,7 +1282,7 @@ public class VmwareStorageProcessor implements StorageProcessor { if (volume.getVmName() == null) { ManagedObjectReference secMorDs = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, volume.getDataStore().getUuid()); DatastoreMO dsMo = new DatastoreMO(hyperHost.getContext(), secMorDs); - workerVmMo = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, "workervm"+volume.getUuid()); + workerVmMo = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, "workervm"+volume.getUuid(), null); if (workerVmMo == null) { throw new Exception("Unable to find created worker VM"); } @@ -1257,7 +1316,7 @@ public class VmwareStorageProcessor implements StorageProcessor { Ternary result = createTemplateFromVolume(context, vmMo, hyperHost, template.getPath(), template.getId(), template.getName(), secondaryStoragePoolURL, volumePath, - hostService.getWorkerName(context, cmd, 0), _nfsVersion); + hostService.getWorkerName(context, cmd, 0, null), _nfsVersion); TemplateObjectTO newTemplate = new TemplateObjectTO(); newTemplate.setPath(result.first()); newTemplate.setFormat(ImageFormat.OVA); @@ -1574,12 +1633,11 @@ public class VmwareStorageProcessor implements StorageProcessor { VmwareContext context = hostService.getServiceContext(cmd); VmwareHypervisorHost hyperHost = hostService.getHyperHost(context, null); - String workerVMName = hostService.getWorkerName(context, cmd, 0); - ManagedObjectReference dsMor = hyperHost.findDatastoreByName(dsFile.getDatastoreName()); DatastoreMO dsMo = new DatastoreMO(context, dsMor); + String workerVMName = hostService.getWorkerName(context, cmd, 0, dsMo); - VirtualMachineMO workerVM = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, workerVMName); + VirtualMachineMO workerVM = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, workerVMName, null); if (workerVM == null) { throw new CloudRuntimeException("Failed to find the newly created worker VM: " + workerVMName); @@ -1833,7 +1891,7 @@ public class VmwareStorageProcessor implements StorageProcessor { } if(vmMo == null) { dsMo = new DatastoreMO(hyperHost.getContext(), morDs); - workerVMName = hostService.getWorkerName(context, cmd, 0); + workerVMName = hostService.getWorkerName(context, cmd, 0, dsMo); vmMo = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, workerVMName, null); if (vmMo == null) { throw new Exception("Failed to find the newly create or relocated VM. vmName: " + workerVMName); @@ -1849,7 +1907,7 @@ public class VmwareStorageProcessor implements StorageProcessor { backupResult = backupSnapshotToSecondaryStorage(context, vmMo, hyperHost, destSnapshot.getPath(), srcSnapshot.getVolume().getPath(), snapshotUuid, secondaryStorageUrl, - prevSnapshotUuid, prevBackupUuid, hostService.getWorkerName(context, cmd, 1), _nfsVersion); + prevSnapshotUuid, prevBackupUuid, hostService.getWorkerName(context, cmd, 1, null), _nfsVersion); snapshotBackupUuid = backupResult.first(); success = (snapshotBackupUuid != null); @@ -2290,10 +2348,10 @@ public class VmwareStorageProcessor implements StorageProcessor { newVol.setSize(volume.getSize()); } catch (Exception e) { s_logger.debug("Create disk using vStorageObject manager failed due to exception " + e.getMessage() + ", retying using worker VM"); - String dummyVmName = hostService.getWorkerName(context, cmd, 0); + String dummyVmName = hostService.getWorkerName(context, cmd, 0, dsMo); try { s_logger.info("Create worker VM " + dummyVmName); - vmMo = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, dummyVmName); + vmMo = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, dummyVmName, null); if (vmMo == null) { throw new Exception("Unable to create a dummy VM for volume creation"); } @@ -2976,9 +3034,9 @@ public class VmwareStorageProcessor implements StorageProcessor { VmwareContext context = hostService.getServiceContext(null); VmwareHypervisorHost hyperHost = hostService.getHyperHost(context, null); - String dummyVmName = hostService.getWorkerName(context, cmd, 0); + String dummyVmName = hostService.getWorkerName(context, cmd, 0, dsMo); - VirtualMachineMO vmMo = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, dummyVmName); + VirtualMachineMO vmMo = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, dummyVmName, null); if (vmMo == null) { throw new Exception("Unable to create a dummy VM for volume creation"); diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java index e8ea88207ac..1ac9636e0e7 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java @@ -1641,6 +1641,9 @@ public class HypervisorHostHelper { if (morCluster != null) hyperHost = new ClusterMO(hyperHost.getContext(), morCluster); + if (dsMo.getDatastoreType().equalsIgnoreCase("VVOL") && !vmName.startsWith(CustomFieldConstants.CLOUD_UUID)) { + vmName = CustomFieldConstants.CLOUD_UUID + "-" + vmName; + } VirtualMachineMO workingVM = null; VirtualMachineConfigSpec vmConfig = new VirtualMachineConfigSpec(); vmConfig.setName(vmName); From f825a94dc459e49f43a5dfc30fec8da563281269 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Thu, 10 Sep 2020 15:00:42 +0530 Subject: [PATCH 080/164] Fixed volume migrations to and fro vVols datastorewq --- .../hypervisor/vmware/resource/VmwareResource.java | 14 +++++++++++--- .../storage/resource/VmwareStorageProcessor.java | 6 +++--- .../cloud/hypervisor/vmware/mo/DatastoreMO.java | 2 ++ .../cloud/hypervisor/vmware/util/VmwareHelper.java | 10 ++++++++-- 4 files changed, 24 insertions(+), 8 deletions(-) diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 54c3b4449af..ad69d89e595 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -4760,7 +4760,9 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa VmwareHypervisorHost hyperHost = getHyperHost(getServiceContext()); VirtualMachineMO vmMo = null; DatastoreMO dsMo = null; + DatastoreMO destinationDsMo = null; ManagedObjectReference morSourceDS = null; + ManagedObjectReference morDestintionDS = null; String vmdkDataStorePath = null; String vmName = null; @@ -4768,15 +4770,21 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa // OfflineVmwareMigration: we need to refactor the worker vm creation out for use in migration methods as well as here // OfflineVmwareMigration: this method is 100 lines and needs refactorring anyway // we need to spawn a worker VM to attach the volume to and move it - vmName = getWorkerName(getServiceContext(), cmd, 0, dsMo); + morSourceDS = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, cmd.getSourcePool().getUuid()); + dsMo = new DatastoreMO(hyperHost.getContext(), morSourceDS); + morDestintionDS = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, cmd.getTargetPool().getUuid()); + destinationDsMo = new DatastoreMO(hyperHost.getContext(), morDestintionDS); + if (dsMo.getDatastoreType().equalsIgnoreCase("VVOL")) + vmName = getWorkerName(getServiceContext(), cmd, 0, dsMo); + + if (destinationDsMo.getDatastoreType().equalsIgnoreCase("VVOL")) + vmName = getWorkerName(getServiceContext(), cmd, 0, destinationDsMo); // OfflineVmwareMigration: refactor for re-use // OfflineVmwareMigration: 1. find data(store) // OfflineVmwareMigration: more robust would be to find the store given the volume as it might have been moved out of band or due to error // example: DatastoreMO existingVmDsMo = new DatastoreMO(dcMo.getContext(), dcMo.findDatastore(fileInDatastore.getDatastoreName())); - morSourceDS = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, cmd.getSourcePool().getUuid()); - dsMo = new DatastoreMO(hyperHost.getContext(), morSourceDS); s_logger.info("Create worker VM " + vmName); // OfflineVmwareMigration: 2. create the worker with access to the data(store) vmMo = HypervisorHostHelper.createWorkerVM(hyperHost, dsMo, vmName, null); diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java index 5ff4ee33808..40a9617cdec 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -989,7 +989,7 @@ public class VmwareStorageProcessor implements StorageProcessor { volumeName = srcVolumePath.substring(index + 1); } - String newVolume = VmwareHelper.getVCenterSafeUuid(); + String newVolume = VmwareHelper.getVCenterSafeUuid(dsMo); restoreVolumeFromSecStorage(hyperHost, dsMo, newVolume, secStorageUrl, volumeFolder, volumeName, wait, nfsVersion); return new Pair<>(volumeFolder, newVolume); @@ -3590,7 +3590,6 @@ public class VmwareStorageProcessor implements StorageProcessor { String backupPath = backedUpSnapshotUuid.substring(0, index); backedUpSnapshotUuid = backedUpSnapshotUuid.substring(index + 1); String details; - String newVolumeName = VmwareHelper.getVCenterSafeUuid(); VmwareContext context = hostService.getServiceContext(cmd); try { @@ -3602,13 +3601,14 @@ public class VmwareStorageProcessor implements StorageProcessor { throw new Exception(msg); } + DatastoreMO primaryDsMo = new DatastoreMO(hyperHost.getContext(), morPrimaryDs); + String newVolumeName = VmwareHelper.getVCenterSafeUuid(primaryDsMo); // strip off the extension since restoreVolumeFromSecStorage internally will append suffix there. if (backedUpSnapshotUuid.endsWith(".ova")){ backedUpSnapshotUuid = backedUpSnapshotUuid.replace(".ova", ""); } else if (backedUpSnapshotUuid.endsWith(".ovf")){ backedUpSnapshotUuid = backedUpSnapshotUuid.replace(".ovf", ""); } - DatastoreMO primaryDsMo = new DatastoreMO(hyperHost.getContext(), morPrimaryDs); restoreVolumeFromSecStorage(hyperHost, primaryDsMo, newVolumeName, secondaryStorageUrl, backupPath, backedUpSnapshotUuid, (long)cmd.getWait() * 1000, _nfsVersion); VolumeObjectTO newVol = new VolumeObjectTO(); diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatastoreMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatastoreMO.java index fb9e83cf308..804af6286d1 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatastoreMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/DatastoreMO.java @@ -406,6 +406,8 @@ public class DatastoreMO extends BaseMO { s_logger.info("Found file " + fileName + " in datastore at " + absoluteFileName); if (parentFolderPath.endsWith("]")) absoluteFileName += " "; + else if (!parentFolderPath.endsWith("/")) + absoluteFileName +="/"; absoluteFileName += fi.getPath(); if(isValidCloudStackFolderPath(parentFolderPath, searchExcludedFolders)) { return absoluteFileName; diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java index babb954eec7..e00bc237018 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java @@ -76,7 +76,9 @@ import com.vmware.vim25.VirtualVmxnet2; import com.vmware.vim25.VirtualVmxnet3; import com.cloud.hypervisor.vmware.mo.DiskControllerType; +import com.cloud.hypervisor.vmware.mo.DatastoreMO; import com.cloud.hypervisor.vmware.mo.HostMO; +import com.cloud.hypervisor.vmware.mo.CustomFieldConstants; import com.cloud.hypervisor.vmware.mo.LicenseAssignmentManagerMO; import com.cloud.hypervisor.vmware.mo.VirtualEthernetCardType; import com.cloud.hypervisor.vmware.mo.VirtualMachineMO; @@ -692,9 +694,13 @@ public class VmwareHelper { return hotplugSupportedByLicense; } - public static String getVCenterSafeUuid() { + public static String getVCenterSafeUuid(DatastoreMO dsMo) throws Exception{ // Object name that is greater than 32 is not safe in vCenter - return UUID.randomUUID().toString().replaceAll("-", ""); + String uuid = UUID.randomUUID().toString().replaceAll("-", ""); + if (dsMo.getDatastoreType().equalsIgnoreCase("VVOL")) { + return CustomFieldConstants.CLOUD_UUID + "-" + uuid; + } + return uuid; } public static String getRecommendedDiskControllerFromDescriptor(GuestOsDescriptor guestOsDescriptor) throws Exception { From f53f11b0103cce567c3aace5d9ee2d8e33b1337f Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Thu, 10 Sep 2020 16:13:11 +0530 Subject: [PATCH 081/164] Fix seeding template issue after PR merging --- .../resource/VmwareStorageProcessor.java | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java index 40a9617cdec..9bbc6e58bda 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -651,25 +651,22 @@ public class VmwareStorageProcessor implements StorageProcessor { VirtualMachineMO templateMo = VmwareHelper.pickOneVmOnRunningHost(dcMo.findVmByNameAndLabel(templateUuidName), true); Pair vmInfo = null; + final ManagedObjectReference morDs; + if (managed) { + morDs = prepareManagedDatastore(context, hyperHost, null, managedStoragePoolName, storageHost, storagePort, + chapInitiatorUsername, chapInitiatorSecret, chapTargetUsername, chapTargetSecret); + } + else { + morDs = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, storageUuid); + } + assert (morDs != null); + dsMo = new DatastoreMO(context, morDs); + if (templateMo == null) { if (s_logger.isInfoEnabled()) { s_logger.info("Template " + templateInfo.second() + " is not setup yet. Set up template from secondary storage with uuid name: " + templateUuidName); } - final ManagedObjectReference morDs; - - if (managed) { - morDs = prepareManagedDatastore(context, hyperHost, null, managedStoragePoolName, storageHost, storagePort, - chapInitiatorUsername, chapInitiatorSecret, chapTargetUsername, chapTargetSecret); - } - else { - morDs = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, storageUuid); - } - - assert (morDs != null); - - dsMo = new DatastoreMO(context, morDs); - if (managed) { vmInfo = copyTemplateFromSecondaryToPrimary(hyperHost, dsMo, secondaryStorageUrl, templateInfo.first(), templateInfo.second(), managedStoragePoolRootVolumeName, false, _nfsVersion, configurationId); From bc8ad6843458aae30f7af2b59cc8547444994ec1 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Fri, 11 Sep 2020 13:08:20 +0530 Subject: [PATCH 082/164] prepare for maintenance of storage pool of type datastore cluster should not try to cancel the maintenance upon failure, rather keep it in error in maintenance state --- .../com/cloud/storage/StorageManagerImpl.java | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index e08e8becc7a..212f6938e65 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -1568,7 +1568,6 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C } } }); - List maintenanceSuccessfulStoragePools = new ArrayList<>(); for (Iterator iteratorChildDatastore = childDatastores.listIterator(); iteratorChildDatastore.hasNext(); ) { DataStore childStore = _dataStoreMgr.getDataStore(iteratorChildDatastore.next().getId(), DataStoreRole.Primary); try { @@ -1576,24 +1575,16 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C } catch (Exception e) { if (s_logger.isDebugEnabled()) { s_logger.debug(String.format("Exception on maintenance preparation of one of the child datastores in datastore cluster %d with error %s", primaryStorageId, e)); - s_logger.debug(String.format("Cancelling the maintenance mode of child datastores in datastore cluster %d", primaryStorageId)); } - // Cancel maintenance mode of already prepared child storage pools - maintenanceSuccessfulStoragePools.add(childStore); - for (DataStore dataStore: maintenanceSuccessfulStoragePools) { - lifeCycle.cancelMaintain(dataStore); - } - // Set back to Up state of remaining child storage pools and datastore cluster - while (iteratorChildDatastore.hasNext()) { - StoragePoolVO childDatastore = iteratorChildDatastore.next(); - childDatastore.setStatus(StoragePoolStatus.Up); + // Set to ErrorInMaintenance state of all child storage pools and datastore cluster + for (StoragePoolVO childDatastore : childDatastores) { + childDatastore.setStatus(StoragePoolStatus.ErrorInMaintenance); _storagePoolDao.update(childDatastore.getId(), childDatastore); } - datastoreCluster.setStatus(StoragePoolStatus.Up); + datastoreCluster.setStatus(StoragePoolStatus.ErrorInMaintenance); _storagePoolDao.update(datastoreCluster.getId(), datastoreCluster); throw new CloudRuntimeException(String.format("Failed to prepare maintenance mode for datastore cluster %d with error %s %s", primaryStorageId, e.getMessage(), e)); } - maintenanceSuccessfulStoragePools.add(childStore); } } From f74262b12a287db863c85a3b05d84824f0c7e82c Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Fri, 11 Sep 2020 17:29:35 +0530 Subject: [PATCH 083/164] Snapshot on dettached volume considering wrong path, corrected it --- .../java/com/cloud/storage/resource/VmwareStorageProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java index 9bbc6e58bda..155a5c5e983 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -1895,7 +1895,7 @@ public class VmwareStorageProcessor implements StorageProcessor { } workerVm = vmMo; // attach volume to worker VM - String datastoreVolumePath = dsMo.getDatastorePath(volumePath + ".vmdk"); + String datastoreVolumePath = VmwareStorageLayoutHelper.getLegacyDatastorePathFromVmdkFileName(dsMo, volumePath + ".vmdk"); vmMo.attachDisk(new String[] { datastoreVolumePath }, morDs); } else { s_logger.info("Using owner VM " + vmName + " for snapshot operation"); From fab6b41c9044d3a5dc3da59cea2ead02bfbdb71f Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Mon, 14 Sep 2020 12:56:10 +0530 Subject: [PATCH 084/164] Delete VM fix which does not move detached disks to fcd folder vVols attach disk fix --- .../vmware/resource/VmwareResource.java | 72 ++++++++++--------- .../resource/VmwareStorageLayoutHelper.java | 4 +- .../resource/VmwareStorageProcessor.java | 20 +++++- 3 files changed, 59 insertions(+), 37 deletions(-) diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index ad69d89e595..810a4d38117 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -2881,43 +2881,51 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } DatastoreMO dsMo = volumeDsDetails.second(); + String datastoreDiskPath; - // we will honor vCenter's meta if it exists - if (diskInfo != null) { - // to deal with run-time upgrade to maintain the new datastore folder structure - String disks[] = diskInfo.getDiskChain(); - for (int i = 0; i < disks.length; i++) { - DatastoreFile file = new DatastoreFile(disks[i]); - if (!isManaged && file.getDir() != null && file.getDir().isEmpty()) { - s_logger.info("Perform run-time datastore folder upgrade. sync " + disks[i] + " to VM folder"); - disks[i] = VmwareStorageLayoutHelper.syncVolumeToVmDefaultFolder(dcMo, vmMo.getName(), dsMo, file.getFileBaseName(), VmwareManager.s_vmwareSearchExcludeFolder.value()); - } + if (dsMo.getDatastoreType().equalsIgnoreCase("VVOL")) { + datastoreDiskPath = VmwareStorageLayoutHelper.getLegacyDatastorePathFromVmdkFileName(dsMo, volumeTO.getPath() + ".vmdk"); + if (!dsMo.fileExists(datastoreDiskPath)) { + datastoreDiskPath = VmwareStorageLayoutHelper.getVmwareDatastorePathFromVmdkFileName(dsMo, vmMo.getName(), volumeTO.getPath() + ".vmdk"); } - return disks; - } - - final String datastoreDiskPath; - - if (isManaged) { - String vmdkPath = new DatastoreFile(volumeTO.getPath()).getFileBaseName(); - - if (volumeTO.getVolumeType() == Volume.Type.ROOT) { - if (vmdkPath == null) { - vmdkPath = volumeTO.getName(); - } - - datastoreDiskPath = VmwareStorageLayoutHelper.syncVolumeToVmDefaultFolder(dcMo, vmMo.getName(), dsMo, vmdkPath); - } else { - if (vmdkPath == null) { - vmdkPath = dsMo.getName(); - } - - datastoreDiskPath = dsMo.getDatastorePath(vmdkPath + VMDK_EXTENSION); + if (!dsMo.fileExists(datastoreDiskPath)) { + datastoreDiskPath = dsMo.searchFileInSubFolders(volumeTO.getPath() + ".vmdk", true, null); } } else { - datastoreDiskPath = VmwareStorageLayoutHelper.syncVolumeToVmDefaultFolder(dcMo, vmMo.getName(), dsMo, volumeTO.getPath(), VmwareManager.s_vmwareSearchExcludeFolder.value()); - } + // we will honor vCenter's meta if it exists + if (diskInfo != null) { + // to deal with run-time upgrade to maintain the new datastore folder structure + String disks[] = diskInfo.getDiskChain(); + for (int i = 0; i < disks.length; i++) { + DatastoreFile file = new DatastoreFile(disks[i]); + if (!isManaged && file.getDir() != null && file.getDir().isEmpty()) { + s_logger.info("Perform run-time datastore folder upgrade. sync " + disks[i] + " to VM folder"); + disks[i] = VmwareStorageLayoutHelper.syncVolumeToVmDefaultFolder(dcMo, vmMo.getName(), dsMo, file.getFileBaseName(), VmwareManager.s_vmwareSearchExcludeFolder.value()); + } + } + return disks; + } + if (isManaged) { + String vmdkPath = new DatastoreFile(volumeTO.getPath()).getFileBaseName(); + + if (volumeTO.getVolumeType() == Volume.Type.ROOT) { + if (vmdkPath == null) { + vmdkPath = volumeTO.getName(); + } + + datastoreDiskPath = VmwareStorageLayoutHelper.syncVolumeToVmDefaultFolder(dcMo, vmMo.getName(), dsMo, vmdkPath); + } else { + if (vmdkPath == null) { + vmdkPath = dsMo.getName(); + } + + datastoreDiskPath = dsMo.getDatastorePath(vmdkPath + VMDK_EXTENSION); + } + } else { + datastoreDiskPath = VmwareStorageLayoutHelper.syncVolumeToVmDefaultFolder(dcMo, vmMo.getName(), dsMo, volumeTO.getPath(), VmwareManager.s_vmwareSearchExcludeFolder.value()); + } + } if (!dsMo.fileExists(datastoreDiskPath)) { s_logger.warn("Volume " + volumeTO.getId() + " does not seem to exist on datastore, out of sync? path: " + datastoreDiskPath); } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageLayoutHelper.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageLayoutHelper.java index 164082c2a73..6c66183240c 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageLayoutHelper.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageLayoutHelper.java @@ -262,8 +262,8 @@ public class VmwareStorageLayoutHelper implements Configurable { s_logger.info("Check if we need to move " + fileFullDsPath + " to its root location"); DatastoreMO dsMo = new DatastoreMO(dcMo.getContext(), dcMo.findDatastore(file.getDatastoreName())); - if (dsMo.getMor() != null) { - DatastoreFile targetFile = new DatastoreFile(file.getDatastoreName(), file.getFileName()); + if (dsMo.getMor() != null && !dsMo.getDatastoreType().equalsIgnoreCase("VVOL")) { + DatastoreFile targetFile = new DatastoreFile(file.getDatastoreName(), HypervisorHostHelper.VSPHERE_DATASTORE_BASE_FOLDER, file.getFileName()); if (!targetFile.getPath().equalsIgnoreCase(file.getPath())) { s_logger.info("Move " + file.getPath() + " -> " + targetFile.getPath()); dsMo.moveDatastoreFile(file.getPath(), dcMo.getMor(), dsMo.getMor(), targetFile.getPath(), dcMo.getMor(), true); diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java index 155a5c5e983..fe9262310e0 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -2086,17 +2086,29 @@ public class VmwareStorageProcessor implements StorageProcessor { if (isManaged) { datastoreVolumePath = dsMo.getDatastorePath((vmdkPath != null ? vmdkPath : dsMo.getName()) + ".vmdk"); } else { - datastoreVolumePath = VmwareStorageLayoutHelper.syncVolumeToVmDefaultFolder(dsMo.getOwnerDatacenter().first(), vmName, dsMo, volumeTO.getPath(), VmwareManager.s_vmwareSearchExcludeFolder.value()); + if (dsMo.getDatastoreType().equalsIgnoreCase("VVOL")) { + datastoreVolumePath = VmwareStorageLayoutHelper.getLegacyDatastorePathFromVmdkFileName(dsMo, volumeTO.getPath() + ".vmdk"); + if (!dsMo.fileExists(datastoreVolumePath)) { + datastoreVolumePath = VmwareStorageLayoutHelper.getVmwareDatastorePathFromVmdkFileName(dsMo, vmName, volumeTO.getPath() + ".vmdk"); + } + if (!dsMo.fileExists(datastoreVolumePath)) { + datastoreVolumePath = dsMo.searchFileInSubFolders(volumeTO.getPath() + ".vmdk", true, null); + } + } else { + datastoreVolumePath = VmwareStorageLayoutHelper.syncVolumeToVmDefaultFolder(dsMo.getOwnerDatacenter().first(), vmName, dsMo, volumeTO.getPath(), VmwareManager.s_vmwareSearchExcludeFolder.value()); + } } } else { if (isManaged) { datastoreVolumePath = dsMo.getDatastorePath((vmdkPath != null ? vmdkPath : dsMo.getName()) + ".vmdk"); } else { datastoreVolumePath = VmwareStorageLayoutHelper.getLegacyDatastorePathFromVmdkFileName(dsMo, volumeTO.getPath() + ".vmdk"); - if (!dsMo.fileExists(datastoreVolumePath)) { datastoreVolumePath = VmwareStorageLayoutHelper.getVmwareDatastorePathFromVmdkFileName(dsMo, vmName, volumeTO.getPath() + ".vmdk"); } + if (!dsMo.fileExists(datastoreVolumePath)) { + datastoreVolumePath = dsMo.searchFileInSubFolders(volumeTO.getPath() + ".vmdk", true, null); + } } } @@ -2127,7 +2139,9 @@ public class VmwareStorageProcessor implements StorageProcessor { if (isManaged) { handleDatastoreAndVmdkDetachManaged(cmd, diskUuid, iScsiName, storageHost, storagePort); } else { - VmwareStorageLayoutHelper.syncVolumeToRootFolder(dsMo.getOwnerDatacenter().first(), dsMo, volumeTO.getPath(), vmName, VmwareManager.s_vmwareSearchExcludeFolder.value()); + if (!dsMo.getDatastoreType().equalsIgnoreCase("VVOL")) { + VmwareStorageLayoutHelper.syncVolumeToRootFolder(dsMo.getOwnerDatacenter().first(), dsMo, volumeTO.getPath(), vmName, VmwareManager.s_vmwareSearchExcludeFolder.value()); + } } } From bb4ce2118ddac69fda923208431a754ae5168dca Mon Sep 17 00:00:00 2001 From: nvazquez Date: Mon, 14 Sep 2020 10:34:01 -0300 Subject: [PATCH 085/164] Add new template and vm deploy as is details table and refactor --- .../cloud/agent/api/storage/OVFHelper.java | 25 ++- .../cloud/agent/api/to/DeployAsIsInfoTO.java | 31 +-- .../cloud/agent/api/to/VirtualMachineTO.java | 11 - .../deployasis}/OVFConfigurationTO.java | 4 +- .../deployasis}/OVFEulaSectionTO.java | 6 +- .../api/to/deployasis/OVFNetworkTO.java} | 4 +- .../deployasis}/OVFPropertyTO.java | 4 +- .../deployasis}/OVFVirtualHardwareItemTO.java | 4 +- .../OVFVirtualHardwareSectionTO.java | 4 +- .../TemplateDeployAsIsInformationTO.java | 24 ++ .../cloud/deployasis/DeployAsIsConstants.java | 27 +++ .../java/com/cloud/storage/ImageStore.java | 7 - .../apache/cloudstack/api/ApiConstants.java | 1 + .../api/response/TemplateResponse.java | 19 ++ .../agent/api/storage/OVFHelperTest.java | 4 + .../agent/api/storage/DownloadAnswer.java | 11 +- .../cloud/storage/template/OVAProcessor.java | 14 +- .../com/cloud/storage/template/Processor.java | 10 +- .../agent/api/storage/DownloadAnswerTest.java | 7 +- .../cloud/vm/VirtualMachineManagerImpl.java | 3 +- .../orchestration/NetworkOrchestrator.java | 8 +- .../TemplateDeployAsIsDetailVO.java | 94 ++++++++ ...ateDeployAsIsDetailVO.java~Stashed changes | 94 ++++++++ .../deployasis/UserVmDeployAsIsDetailVO.java | 94 ++++++++ ...rVmDeployAsIsDetailVO.java~Stashed changes | 94 ++++++++ .../dao/TemplateDeployAsIsDetailsDao.java | 32 +++ .../dao/TemplateDeployAsIsDetailsDaoImpl.java | 83 +++++++ ...loyAsIsDetailsDaoImpl.java~Stashed changes | 83 +++++++ .../dao/UserVmDeployAsIsDetailsDao.java | 24 ++ .../dao/UserVmDeployAsIsDetailsDaoImpl.java | 30 +++ .../com/cloud/storage/VMTemplateDetailVO.java | 4 +- .../storage/dao/VMTemplateDetailsDao.java | 14 -- .../storage/dao/VMTemplateDetailsDaoImpl.java | 124 +--------- .../java/com/cloud/vm/dao/UserVmDaoImpl.java | 15 +- ...spring-engine-schema-core-daos-context.xml | 2 + .../META-INF/db/schema-41400to41500.sql | 86 +++++++ .../image/BaseImageStoreDriverImpl.java | 195 +--------------- .../image/deployasis/DeployAsIsHelper.java | 15 ++ .../deployasis/DeployAsIsHelperImpl.java | 212 ++++++++++++++++++ .../spring-engine-storage-core-context.xml | 2 + .../hypervisor/guru/VmwareVmImplementer.java | 115 ++-------- .../vmware/resource/VmwareResource.java | 76 ++----- .../api/query/dao/TemplateJoinDaoImpl.java | 20 ++ .../java/com/cloud/vm/UserVmManagerImpl.java | 70 ++---- .../storage/template/DownloadManagerImpl.java | 14 +- 45 files changed, 1190 insertions(+), 630 deletions(-) rename api/src/main/java/com/cloud/agent/api/{storage => to/deployasis}/OVFConfigurationTO.java (93%) rename api/src/main/java/com/cloud/agent/api/{storage => to/deployasis}/OVFEulaSectionTO.java (91%) rename api/src/main/java/{org/apache/cloudstack/api/net/NetworkPrerequisiteTO.java => com/cloud/agent/api/to/deployasis/OVFNetworkTO.java} (96%) rename api/src/main/java/com/cloud/agent/api/{storage => to/deployasis}/OVFPropertyTO.java (97%) rename api/src/main/java/com/cloud/agent/api/{storage => to/deployasis}/OVFVirtualHardwareItemTO.java (98%) rename api/src/main/java/com/cloud/agent/api/{storage => to/deployasis}/OVFVirtualHardwareSectionTO.java (91%) create mode 100644 api/src/main/java/com/cloud/agent/api/to/deployasis/TemplateDeployAsIsInformationTO.java create mode 100644 api/src/main/java/com/cloud/deployasis/DeployAsIsConstants.java create mode 100644 engine/schema/src/main/java/com/cloud/deployasis/TemplateDeployAsIsDetailVO.java create mode 100644 engine/schema/src/main/java/com/cloud/deployasis/TemplateDeployAsIsDetailVO.java~Stashed changes create mode 100644 engine/schema/src/main/java/com/cloud/deployasis/UserVmDeployAsIsDetailVO.java create mode 100644 engine/schema/src/main/java/com/cloud/deployasis/UserVmDeployAsIsDetailVO.java~Stashed changes create mode 100644 engine/schema/src/main/java/com/cloud/deployasis/dao/TemplateDeployAsIsDetailsDao.java create mode 100644 engine/schema/src/main/java/com/cloud/deployasis/dao/TemplateDeployAsIsDetailsDaoImpl.java create mode 100644 engine/schema/src/main/java/com/cloud/deployasis/dao/TemplateDeployAsIsDetailsDaoImpl.java~Stashed changes create mode 100644 engine/schema/src/main/java/com/cloud/deployasis/dao/UserVmDeployAsIsDetailsDao.java create mode 100644 engine/schema/src/main/java/com/cloud/deployasis/dao/UserVmDeployAsIsDetailsDaoImpl.java create mode 100644 engine/storage/src/main/java/org/apache/cloudstack/storage/image/deployasis/DeployAsIsHelper.java create mode 100644 engine/storage/src/main/java/org/apache/cloudstack/storage/image/deployasis/DeployAsIsHelperImpl.java diff --git a/api/src/main/java/com/cloud/agent/api/storage/OVFHelper.java b/api/src/main/java/com/cloud/agent/api/storage/OVFHelper.java index bd58a189f76..45eff89b0c9 100644 --- a/api/src/main/java/com/cloud/agent/api/storage/OVFHelper.java +++ b/api/src/main/java/com/cloud/agent/api/storage/OVFHelper.java @@ -39,10 +39,15 @@ import javax.xml.transform.TransformerFactory; import javax.xml.transform.dom.DOMSource; import javax.xml.transform.stream.StreamResult; +import com.cloud.agent.api.to.deployasis.OVFConfigurationTO; +import com.cloud.agent.api.to.deployasis.OVFEulaSectionTO; +import com.cloud.agent.api.to.deployasis.OVFPropertyTO; +import com.cloud.agent.api.to.deployasis.OVFVirtualHardwareItemTO; +import com.cloud.agent.api.to.deployasis.OVFVirtualHardwareSectionTO; import com.cloud.configuration.Resource.ResourceType; import com.cloud.exception.InternalErrorException; import com.cloud.utils.compression.CompressionUtil; -import org.apache.cloudstack.api.net.NetworkPrerequisiteTO; +import com.cloud.agent.api.to.deployasis.OVFNetworkTO; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.math.NumberUtils; @@ -573,7 +578,7 @@ public class OVFHelper { return null; } - public List getNetPrerequisitesFromDocument(Document doc) throws InternalErrorException { + public List getNetPrerequisitesFromDocument(Document doc) throws InternalErrorException { if (doc == null) { if (s_logger.isTraceEnabled()) { s_logger.trace("no document to parse; returning no prerequiste networks"); @@ -581,7 +586,7 @@ public class OVFHelper { return Collections.emptyList(); } - Map nets = getNetworksFromDocumentTree(doc); + Map nets = getNetworksFromDocumentTree(doc); checkForOnlyOneSystemNode(doc); @@ -590,7 +595,7 @@ public class OVFHelper { return new ArrayList<>(nets.values()); } - private void matchNicsToNets(Map nets, Node systemElement) { + private void matchNicsToNets(Map nets, Node systemElement) { final DocumentTraversal traversal = (DocumentTraversal) systemElement; final NodeIterator iterator = traversal.createNodeIterator(systemElement, NodeFilter.SHOW_ELEMENT, null, true); if (s_logger.isTraceEnabled()) { @@ -606,9 +611,9 @@ public class OVFHelper { if(s_logger.isInfoEnabled()) { s_logger.info(String.format("found a nic definition without a network definition byname %s, adding it to the list.", name)); } - nets.put(name, new NetworkPrerequisiteTO()); + nets.put(name, new OVFNetworkTO()); } - NetworkPrerequisiteTO thisNet = nets.get(name); + OVFNetworkTO thisNet = nets.get(name); if (e.getParentNode() != null) { fillNicPrerequisites(thisNet,e.getParentNode()); } @@ -625,7 +630,7 @@ public class OVFHelper { * @param nic the object to carry through the system * @param parentNode the xml container node for nic data */ - private void fillNicPrerequisites(NetworkPrerequisiteTO nic, Node parentNode) { + private void fillNicPrerequisites(OVFNetworkTO nic, Node parentNode) { String addressOnParentStr = getChildNodeValue(parentNode, "AddressOnParent"); String automaticAllocationStr = getChildNodeValue(parentNode, "AutomaticAllocation"); String description = getChildNodeValue(parentNode, "Description"); @@ -667,9 +672,9 @@ public class OVFHelper { } } - private Map getNetworksFromDocumentTree(Document doc) { + private Map getNetworksFromDocumentTree(Document doc) { NodeList networkElements = doc.getElementsByTagName("Network"); - Map nets = new HashMap<>(); + Map nets = new HashMap<>(); for (int i = 0; i < networkElements.getLength(); i++) { Element networkElement = (Element)networkElements.item(i); @@ -677,7 +682,7 @@ public class OVFHelper { String description = getChildNodeValue(networkElement, "Description"); - NetworkPrerequisiteTO network = new NetworkPrerequisiteTO(); + OVFNetworkTO network = new OVFNetworkTO(); network.setName(networkName); network.setNetworkDescription(description); diff --git a/api/src/main/java/com/cloud/agent/api/to/DeployAsIsInfoTO.java b/api/src/main/java/com/cloud/agent/api/to/DeployAsIsInfoTO.java index c51c2632137..cc0e1d01d03 100644 --- a/api/src/main/java/com/cloud/agent/api/to/DeployAsIsInfoTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/DeployAsIsInfoTO.java @@ -16,36 +16,37 @@ // under the License. package com.cloud.agent.api.to; +import com.cloud.agent.api.LogLevel; + +import java.util.HashMap; +import java.util.Map; + public class DeployAsIsInfoTO { - private boolean deployAsIs; private String templatePath; - private String deploymentConfiguration; + private String destStoragePool; + @LogLevel(LogLevel.Log4jLevel.Off) + private Map properties = new HashMap<>(); public DeployAsIsInfoTO() { } - public boolean isDeployAsIs() { - return deployAsIs; - } - - public void setDeployAsIs(boolean deployAsIs) { - this.deployAsIs = deployAsIs; + public DeployAsIsInfoTO(String templatePath, String destStoragePool, Map properties) { + this.templatePath = templatePath; + this.destStoragePool = destStoragePool; + this.properties = properties; } public String getTemplatePath() { return templatePath; } - public void setTemplatePath(String templateInSecondaryPath) { - this.templatePath = templateInSecondaryPath; + public Map getProperties() { + return properties; } - public String getDeploymentConfiguration() { - return deploymentConfiguration; + public String getDestStoragePool() { + return destStoragePool; } - public void setDeploymentConfiguration(String deploymentConfiguration) { - this.deploymentConfiguration = deploymentConfiguration; - } } diff --git a/api/src/main/java/com/cloud/agent/api/to/VirtualMachineTO.java b/api/src/main/java/com/cloud/agent/api/to/VirtualMachineTO.java index 7dbc70d6629..efc735ccecf 100644 --- a/api/src/main/java/com/cloud/agent/api/to/VirtualMachineTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/VirtualMachineTO.java @@ -20,8 +20,6 @@ import java.util.List; import java.util.Map; import java.util.HashMap; -import com.cloud.agent.api.LogLevel; -import com.cloud.agent.api.storage.OVFPropertyTO; import com.cloud.template.VirtualMachineTemplate.BootloaderType; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachine.Type; @@ -80,8 +78,6 @@ public class VirtualMachineTO { Map guestOsDetails = new HashMap(); Map extraConfig = new HashMap<>(); - @LogLevel(LogLevel.Log4jLevel.Off) - List ovfProperties; DeployAsIsInfoTO deployAsIsInfo; public VirtualMachineTO(long id, String instanceName, VirtualMachine.Type type, int cpus, Integer speed, long minRam, long maxRam, BootloaderType bootloader, @@ -376,13 +372,6 @@ public class VirtualMachineTO { return extraConfig; } - public List getOvfProperties() { - return ovfProperties; - } - - public void setOvfProperties(List ovfProperties) { - this.ovfProperties = ovfProperties; - } public String getBootType() { return bootType; } diff --git a/api/src/main/java/com/cloud/agent/api/storage/OVFConfigurationTO.java b/api/src/main/java/com/cloud/agent/api/to/deployasis/OVFConfigurationTO.java similarity index 93% rename from api/src/main/java/com/cloud/agent/api/storage/OVFConfigurationTO.java rename to api/src/main/java/com/cloud/agent/api/to/deployasis/OVFConfigurationTO.java index 983a8b808f4..f3cb75089ab 100644 --- a/api/src/main/java/com/cloud/agent/api/storage/OVFConfigurationTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/deployasis/OVFConfigurationTO.java @@ -16,11 +16,11 @@ // specific language governing permissions and limitations // under the License. // -package com.cloud.agent.api.storage; +package com.cloud.agent.api.to.deployasis; import java.util.List; -public class OVFConfigurationTO { +public class OVFConfigurationTO implements TemplateDeployAsIsInformationTO { private final String id; private final String label; diff --git a/api/src/main/java/com/cloud/agent/api/storage/OVFEulaSectionTO.java b/api/src/main/java/com/cloud/agent/api/to/deployasis/OVFEulaSectionTO.java similarity index 91% rename from api/src/main/java/com/cloud/agent/api/storage/OVFEulaSectionTO.java rename to api/src/main/java/com/cloud/agent/api/to/deployasis/OVFEulaSectionTO.java index eff478c88b3..893661782f4 100644 --- a/api/src/main/java/com/cloud/agent/api/storage/OVFEulaSectionTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/deployasis/OVFEulaSectionTO.java @@ -16,16 +16,14 @@ // specific language governing permissions and limitations // under the License. // -package com.cloud.agent.api.storage; +package com.cloud.agent.api.to.deployasis; import com.cloud.agent.api.LogLevel; -import java.io.Serializable; - /** * End-user licence agreement */ -public class OVFEulaSectionTO implements Serializable { +public class OVFEulaSectionTO implements TemplateDeployAsIsInformationTO { private String info; @LogLevel(LogLevel.Log4jLevel.Off) private byte[] compressedLicense; diff --git a/api/src/main/java/org/apache/cloudstack/api/net/NetworkPrerequisiteTO.java b/api/src/main/java/com/cloud/agent/api/to/deployasis/OVFNetworkTO.java similarity index 96% rename from api/src/main/java/org/apache/cloudstack/api/net/NetworkPrerequisiteTO.java rename to api/src/main/java/com/cloud/agent/api/to/deployasis/OVFNetworkTO.java index e39849685de..9b05dbc1863 100644 --- a/api/src/main/java/org/apache/cloudstack/api/net/NetworkPrerequisiteTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/deployasis/OVFNetworkTO.java @@ -14,7 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -package org.apache.cloudstack.api.net; +package com.cloud.agent.api.to.deployasis; /** * container for the network prerequisites as found in the appliance template @@ -38,7 +38,7 @@ package org.apache.cloudstack.api.net; * * {code} */ -public class NetworkPrerequisiteTO { +public class OVFNetworkTO implements TemplateDeployAsIsInformationTO { String name; String networkDescription; diff --git a/api/src/main/java/com/cloud/agent/api/storage/OVFPropertyTO.java b/api/src/main/java/com/cloud/agent/api/to/deployasis/OVFPropertyTO.java similarity index 97% rename from api/src/main/java/com/cloud/agent/api/storage/OVFPropertyTO.java rename to api/src/main/java/com/cloud/agent/api/to/deployasis/OVFPropertyTO.java index 24207e92821..32c62559262 100644 --- a/api/src/main/java/com/cloud/agent/api/storage/OVFPropertyTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/deployasis/OVFPropertyTO.java @@ -17,7 +17,7 @@ // under the License. // -package com.cloud.agent.api.storage; +package com.cloud.agent.api.to.deployasis; import com.cloud.agent.api.LogLevel; @@ -30,7 +30,7 @@ import com.cloud.agent.api.LogLevel; * Choose "Remote HTTP and SSH Client Routes" to route only traffic destined for the management client(s), when they are on remote networks. * */ -public class OVFPropertyTO { +public class OVFPropertyTO implements TemplateDeployAsIsInformationTO { private String key; private String type; diff --git a/api/src/main/java/com/cloud/agent/api/storage/OVFVirtualHardwareItemTO.java b/api/src/main/java/com/cloud/agent/api/to/deployasis/OVFVirtualHardwareItemTO.java similarity index 98% rename from api/src/main/java/com/cloud/agent/api/storage/OVFVirtualHardwareItemTO.java rename to api/src/main/java/com/cloud/agent/api/to/deployasis/OVFVirtualHardwareItemTO.java index 365ef7d37b5..52577506653 100644 --- a/api/src/main/java/com/cloud/agent/api/storage/OVFVirtualHardwareItemTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/deployasis/OVFVirtualHardwareItemTO.java @@ -14,10 +14,10 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -package com.cloud.agent.api.storage; +package com.cloud.agent.api.to.deployasis; // From: https://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData.xsd -public class OVFVirtualHardwareItemTO { +public class OVFVirtualHardwareItemTO implements TemplateDeployAsIsInformationTO{ //From: https://schemas.dmtf.org/wbem/cim-html/2/CIM_ResourceAllocationSettingData.html public enum HardwareResourceType { diff --git a/api/src/main/java/com/cloud/agent/api/storage/OVFVirtualHardwareSectionTO.java b/api/src/main/java/com/cloud/agent/api/to/deployasis/OVFVirtualHardwareSectionTO.java similarity index 91% rename from api/src/main/java/com/cloud/agent/api/storage/OVFVirtualHardwareSectionTO.java rename to api/src/main/java/com/cloud/agent/api/to/deployasis/OVFVirtualHardwareSectionTO.java index 9a3ae1244eb..2698f8c5e18 100644 --- a/api/src/main/java/com/cloud/agent/api/storage/OVFVirtualHardwareSectionTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/deployasis/OVFVirtualHardwareSectionTO.java @@ -16,11 +16,11 @@ // specific language governing permissions and limitations // under the License. // -package com.cloud.agent.api.storage; +package com.cloud.agent.api.to.deployasis; import java.util.List; -public class OVFVirtualHardwareSectionTO { +public class OVFVirtualHardwareSectionTO implements TemplateDeployAsIsInformationTO { public OVFVirtualHardwareSectionTO() { } diff --git a/api/src/main/java/com/cloud/agent/api/to/deployasis/TemplateDeployAsIsInformationTO.java b/api/src/main/java/com/cloud/agent/api/to/deployasis/TemplateDeployAsIsInformationTO.java new file mode 100644 index 00000000000..9080b92f594 --- /dev/null +++ b/api/src/main/java/com/cloud/agent/api/to/deployasis/TemplateDeployAsIsInformationTO.java @@ -0,0 +1,24 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package com.cloud.agent.api.to.deployasis; + +import java.io.Serializable; + +public interface TemplateDeployAsIsInformationTO extends Serializable { +} diff --git a/api/src/main/java/com/cloud/deployasis/DeployAsIsConstants.java b/api/src/main/java/com/cloud/deployasis/DeployAsIsConstants.java new file mode 100644 index 00000000000..8c50a3ff401 --- /dev/null +++ b/api/src/main/java/com/cloud/deployasis/DeployAsIsConstants.java @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.deployasis; + +public interface DeployAsIsConstants { + + String ACS_PROPERTY_PREFIX = "ACS-property-"; + String REQUIRED_NETWORK_PREFIX = "ACS-network-"; + String OVF_HARDWARE_CONFIGURATION_PREFIX = "ACS-configuration-"; + String OVF_HARDWARE_ITEM_PREFIX = "ACS-hardware-item-"; + String OVF_EULA_SECTION_PREFIX = "ACS-eula-"; + +} diff --git a/api/src/main/java/com/cloud/storage/ImageStore.java b/api/src/main/java/com/cloud/storage/ImageStore.java index 017f367c13e..c019b17421d 100644 --- a/api/src/main/java/com/cloud/storage/ImageStore.java +++ b/api/src/main/java/com/cloud/storage/ImageStore.java @@ -21,13 +21,6 @@ import org.apache.cloudstack.api.InternalIdentity; public interface ImageStore extends Identity, InternalIdentity { - String ACS_PROPERTY_PREFIX = "ACS-property-"; - String REQUIRED_NETWORK_PREFIX = "ACS-network-"; - String DISK_DEFINITION_PREFIX = "ACS-disk-"; - String OVF_HARDWARE_CONFIGURATION_PREFIX = "ACS-configuration-"; - String OVF_HARDWARE_ITEM_PREFIX = "ACS-hardware-item-"; - String OVF_EULA_SECTION_PREFIX = "ACS-eula-"; - /** * @return name of the object store. */ diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 4751e312f92..ba1f1768395 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -826,6 +826,7 @@ public class ApiConstants { public static final String BOOT_MODE = "bootmode"; public static final String BOOT_INTO_SETUP = "bootintosetup"; public static final String DEPLOY_AS_IS = "deployasis"; + public static final String DEPLOY_AS_IS_DETAILS = "deployasisdetails"; public static final String CROSS_ZONES = "crossZones"; public static final String TEMPLATETYPE = "templatetype"; public static final String SOURCETEMPLATEID = "sourcetemplateid"; diff --git a/api/src/main/java/org/apache/cloudstack/api/response/TemplateResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/TemplateResponse.java index 457d29c7ab1..c733f37a9fb 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/TemplateResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/TemplateResponse.java @@ -199,6 +199,10 @@ public class TemplateResponse extends BaseResponseWithTagInformation implements @Param(description = "VMware only: true if template is deployed without orchestrating disks and networks but \"as-is\" defined in the template.") private Boolean deployAsIs; + @SerializedName(ApiConstants.DEPLOY_AS_IS_DETAILS) + @Param(description = "VMware only: additional key/value details tied with deploy-as-is template") + private Map deployAsIsDetails; + @SerializedName("parenttemplateid") @Param(description = "if Datadisk template, then id of the root disk template this template belongs to") @Deprecated(since = "4.15") @@ -429,4 +433,19 @@ public class TemplateResponse extends BaseResponseWithTagInformation implements public void setRequiresHvm(Boolean requiresHvm) { this.requiresHvm = requiresHvm; } + + public Map getDeployAsIsDetails() { + return this.deployAsIsDetails; + } + + public void setDeployAsIsDetails(Map details) { + this.deployAsIsDetails = details; + } + + public void addDeployAsIsDetail(String key, String value) { + if (this.deployAsIsDetails == null) { + setDeployAsIsDetails(new HashMap<>()); + } + this.deployAsIsDetails.put(key,value); + } } diff --git a/api/src/test/java/com/cloud/agent/api/storage/OVFHelperTest.java b/api/src/test/java/com/cloud/agent/api/storage/OVFHelperTest.java index c4f71052496..e28680784d7 100644 --- a/api/src/test/java/com/cloud/agent/api/storage/OVFHelperTest.java +++ b/api/src/test/java/com/cloud/agent/api/storage/OVFHelperTest.java @@ -16,6 +16,10 @@ // under the License. package com.cloud.agent.api.storage; +import com.cloud.agent.api.to.deployasis.OVFConfigurationTO; +import com.cloud.agent.api.to.deployasis.OVFEulaSectionTO; +import com.cloud.agent.api.to.deployasis.OVFPropertyTO; +import com.cloud.agent.api.to.deployasis.OVFVirtualHardwareItemTO; import org.junit.Assert; import org.junit.Test; import org.xml.sax.SAXException; diff --git a/core/src/main/java/com/cloud/agent/api/storage/DownloadAnswer.java b/core/src/main/java/com/cloud/agent/api/storage/DownloadAnswer.java index a7b9179e200..9ae1d6dcca5 100644 --- a/core/src/main/java/com/cloud/agent/api/storage/DownloadAnswer.java +++ b/core/src/main/java/com/cloud/agent/api/storage/DownloadAnswer.java @@ -26,9 +26,12 @@ import com.cloud.agent.api.Answer; import com.cloud.agent.api.Command; import com.cloud.agent.api.LogLevel; import com.cloud.agent.api.to.DatadiskTO; +import com.cloud.agent.api.to.deployasis.OVFEulaSectionTO; +import com.cloud.agent.api.to.deployasis.OVFPropertyTO; +import com.cloud.agent.api.to.deployasis.OVFVirtualHardwareSectionTO; import com.cloud.storage.VMTemplateStorageResourceAssoc; import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; -import org.apache.cloudstack.api.net.NetworkPrerequisiteTO; +import com.cloud.agent.api.to.deployasis.OVFNetworkTO; public class DownloadAnswer extends Answer { private String jobId; @@ -44,7 +47,7 @@ public class DownloadAnswer extends Answer { @LogLevel(LogLevel.Log4jLevel.Off) private List ovfProperties; @LogLevel(LogLevel.Log4jLevel.Off) - private List networkRequirements; + private List networkRequirements; @LogLevel(LogLevel.Log4jLevel.Off) private List disks; @LogLevel(LogLevel.Log4jLevel.Off) @@ -169,11 +172,11 @@ public class DownloadAnswer extends Answer { this.ovfProperties = ovfProperties; } - public List getNetworkRequirements() { + public List getNetworkRequirements() { return networkRequirements; } - public void setNetworkRequirements(List networkRequirements) { + public void setNetworkRequirements(List networkRequirements) { this.networkRequirements = networkRequirements; } diff --git a/core/src/main/java/com/cloud/storage/template/OVAProcessor.java b/core/src/main/java/com/cloud/storage/template/OVAProcessor.java index 794ac48c904..e980293b750 100644 --- a/core/src/main/java/com/cloud/storage/template/OVAProcessor.java +++ b/core/src/main/java/com/cloud/storage/template/OVAProcessor.java @@ -28,12 +28,12 @@ import javax.naming.ConfigurationException; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; -import com.cloud.agent.api.storage.OVFConfigurationTO; -import com.cloud.agent.api.storage.OVFEulaSectionTO; -import com.cloud.agent.api.storage.OVFPropertyTO; -import com.cloud.agent.api.storage.OVFVirtualHardwareItemTO; -import com.cloud.agent.api.storage.OVFVirtualHardwareSectionTO; -import org.apache.cloudstack.api.net.NetworkPrerequisiteTO; +import com.cloud.agent.api.to.deployasis.OVFConfigurationTO; +import com.cloud.agent.api.to.deployasis.OVFEulaSectionTO; +import com.cloud.agent.api.to.deployasis.OVFPropertyTO; +import com.cloud.agent.api.to.deployasis.OVFVirtualHardwareItemTO; +import com.cloud.agent.api.to.deployasis.OVFVirtualHardwareSectionTO; +import com.cloud.agent.api.to.deployasis.OVFNetworkTO; import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; import org.w3c.dom.Document; @@ -115,7 +115,7 @@ public class OVAProcessor extends AdapterBase implements Processor { info.disks = disks; } - List nets = ovfHelper.getNetPrerequisitesFromDocument(doc); + List nets = ovfHelper.getNetPrerequisitesFromDocument(doc); if (CollectionUtils.isNotEmpty(nets)) { LOGGER.info("Found " + nets.size() + " prerequisite networks"); info.networks = nets; diff --git a/core/src/main/java/com/cloud/storage/template/Processor.java b/core/src/main/java/com/cloud/storage/template/Processor.java index 53fa6b76d04..15a3dec0c6e 100644 --- a/core/src/main/java/com/cloud/storage/template/Processor.java +++ b/core/src/main/java/com/cloud/storage/template/Processor.java @@ -23,14 +23,14 @@ import java.io.File; import java.io.IOException; import java.util.List; -import com.cloud.agent.api.storage.OVFEulaSectionTO; -import com.cloud.agent.api.storage.OVFPropertyTO; -import com.cloud.agent.api.storage.OVFVirtualHardwareSectionTO; +import com.cloud.agent.api.to.deployasis.OVFEulaSectionTO; +import com.cloud.agent.api.to.deployasis.OVFPropertyTO; +import com.cloud.agent.api.to.deployasis.OVFVirtualHardwareSectionTO; import com.cloud.agent.api.to.DatadiskTO; import com.cloud.exception.InternalErrorException; import com.cloud.storage.Storage.ImageFormat; import com.cloud.utils.component.Adapter; -import org.apache.cloudstack.api.net.NetworkPrerequisiteTO; +import com.cloud.agent.api.to.deployasis.OVFNetworkTO; /** * Generic interface to process different types of image formats @@ -59,7 +59,7 @@ public interface Processor extends Adapter { public String filename; public boolean isCorrupted; public List ovfProperties; - public List networks; + public List networks; public List disks; public OVFVirtualHardwareSectionTO hardwareSection; public List eulaSections; diff --git a/core/src/test/java/com/cloud/agent/api/storage/DownloadAnswerTest.java b/core/src/test/java/com/cloud/agent/api/storage/DownloadAnswerTest.java index 62bb3d65c83..c7dcc22572a 100644 --- a/core/src/test/java/com/cloud/agent/api/storage/DownloadAnswerTest.java +++ b/core/src/test/java/com/cloud/agent/api/storage/DownloadAnswerTest.java @@ -17,10 +17,11 @@ package com.cloud.agent.api.storage; import com.cloud.agent.api.Answer; +import com.cloud.agent.api.to.deployasis.OVFPropertyTO; import com.cloud.serializer.GsonHelper; import com.cloud.storage.VMTemplateStorageResourceAssoc; import com.google.gson.Gson; -import org.apache.cloudstack.api.net.NetworkPrerequisiteTO; +import com.cloud.agent.api.to.deployasis.OVFNetworkTO; import org.junit.Assert; import org.junit.Test; @@ -45,8 +46,8 @@ public class DownloadAnswerTest { { List properties = new ArrayList<>(); properties.add(new OVFPropertyTO()); - List networks = new ArrayList<>(); - networks.add(new NetworkPrerequisiteTO()); + List networks = new ArrayList<>(); + networks.add(new OVFNetworkTO()); answer.setOvfProperties(properties); answer.setNetworkRequirements(networks); diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index fb4fadd9500..7ed090db9ff 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -1472,8 +1472,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac if (disk.getType() != Volume.Type.ISO) { final VolumeObjectTO vol = (VolumeObjectTO)disk.getData(); final VolumeVO volume = _volsDao.findById(vol.getId()); - if (vmSpec.getDeployAsIsInfo() != null && vmSpec.getDeployAsIsInfo().isDeployAsIs() - && StringUtils.isNotBlank(vol.getPath())) { + if (vmSpec.getDeployAsIsInfo() != null && StringUtils.isNotBlank(vol.getPath())) { volume.setPath(vol.getPath()); _volsDao.update(volume.getId(), volume); } diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java index 5b635bd2255..892ae2bac5c 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java @@ -40,9 +40,10 @@ import java.util.stream.Collectors; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.deployasis.dao.TemplateDeployAsIsDetailsDao; import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.net.NetworkPrerequisiteTO; +import com.cloud.agent.api.to.deployasis.OVFNetworkTO; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.cloud.entity.api.db.VMNetworkMapVO; import org.apache.cloudstack.engine.cloud.entity.api.db.dao.VMNetworkMapDao; @@ -181,7 +182,6 @@ import com.cloud.offerings.NetworkOfferingVO; import com.cloud.offerings.dao.NetworkOfferingDao; import com.cloud.offerings.dao.NetworkOfferingDetailsDao; import com.cloud.offerings.dao.NetworkOfferingServiceMapDao; -import com.cloud.storage.dao.VMTemplateDetailsDao; import com.cloud.user.Account; import com.cloud.user.ResourceLimitService; import com.cloud.user.User; @@ -304,7 +304,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra @Inject UserVmManager _userVmMgr; @Inject - VMTemplateDetailsDao templateDetailsDao; + TemplateDeployAsIsDetailsDao templateDeployAsIsDetailsDao; List networkGurus; @@ -884,7 +884,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra } } - List netprereqs = templateDetailsDao.listNetworkRequirementsByTemplateId(vm.getTemplate().getId()); + List netprereqs = templateDeployAsIsDetailsDao.listNetworkRequirementsByTemplateId(vm.getTemplate().getId()); if (size < netprereqs.size()) { size = netprereqs.size(); } diff --git a/engine/schema/src/main/java/com/cloud/deployasis/TemplateDeployAsIsDetailVO.java b/engine/schema/src/main/java/com/cloud/deployasis/TemplateDeployAsIsDetailVO.java new file mode 100644 index 00000000000..00a69cd0b0c --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/deployasis/TemplateDeployAsIsDetailVO.java @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.deployasis; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Lob; +import javax.persistence.Table; + +import org.apache.cloudstack.api.ResourceDetail; + +@Entity +@Table(name = "template_deploy_as_is_details") +public class TemplateDeployAsIsDetailVO implements ResourceDetail { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + private long id; + + @Column(name = "template_id") + private long templateId; + + @Column(name = "name") + private String name; + + @Lob + @Column(name = "value", length = 65535) + private String value; + + public TemplateDeployAsIsDetailVO() { + } + + public TemplateDeployAsIsDetailVO(long templateId, String name, String value) { + this.templateId = templateId; + this.name = name; + this.value = value; + } + + @Override + public long getId() { + return id; + } + + public long getResourceId() { + return templateId; + } + + public String getName() { + return name; + } + + public String getValue() { + return value; + } + + @Override + public boolean isDisplay() { + return true; + } + + public void setId(long id) { + this.id = id; + } + + public void setTemplateId(long resourceId) { + this.templateId = resourceId; + } + + public void setName(String name) { + this.name = name; + } + + public void setValue(String value) { + this.value = value; + } +} diff --git a/engine/schema/src/main/java/com/cloud/deployasis/TemplateDeployAsIsDetailVO.java~Stashed changes b/engine/schema/src/main/java/com/cloud/deployasis/TemplateDeployAsIsDetailVO.java~Stashed changes new file mode 100644 index 00000000000..047d985d0bb --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/deployasis/TemplateDeployAsIsDetailVO.java~Stashed changes @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.deployasis; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Lob; +import javax.persistence.Table; + +import org.apache.cloudstack.api.ResourceDetail; + +@Entity +@Table(name = "template_deploy_as_is_details") +public class TemplateDeployAsIsDetailVO implements ResourceDetail { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + private long id; + + @Column(name = "template_id") + private long resourceId; + + @Column(name = "name") + private String name; + + @Lob + @Column(name = "value", length = 65535) + private String value; + + public TemplateDeployAsIsDetailVO() { + } + + public TemplateDeployAsIsDetailVO(long templateId, String name, String value) { + this.resourceId = templateId; + this.name = name; + this.value = value; + } + + @Override + public long getId() { + return id; + } + + public long getResourceId() { + return resourceId; + } + + public String getName() { + return name; + } + + public String getValue() { + return value; + } + + @Override + public boolean isDisplay() { + return true; + } + + public void setId(long id) { + this.id = id; + } + + public void setResourceId(long resourceId) { + this.resourceId = resourceId; + } + + public void setName(String name) { + this.name = name; + } + + public void setValue(String value) { + this.value = value; + } +} diff --git a/engine/schema/src/main/java/com/cloud/deployasis/UserVmDeployAsIsDetailVO.java b/engine/schema/src/main/java/com/cloud/deployasis/UserVmDeployAsIsDetailVO.java new file mode 100644 index 00000000000..6cbc22b6b38 --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/deployasis/UserVmDeployAsIsDetailVO.java @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.deployasis; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Lob; +import javax.persistence.Table; + +import org.apache.cloudstack.api.ResourceDetail; + +@Entity +@Table(name = "user_vm_deploy_as_is_details") +public class UserVmDeployAsIsDetailVO implements ResourceDetail { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + private long id; + + @Column(name = "vm_id") + private long vmId; + + @Column(name = "name") + private String name; + + @Lob + @Column(name = "value", length = 65535) + private String value; + + public UserVmDeployAsIsDetailVO() { + } + + public UserVmDeployAsIsDetailVO(long vmId, String name, String value) { + this.vmId = vmId; + this.name = name; + this.value = value; + } + + @Override + public long getId() { + return id; + } + + public long getResourceId() { + return vmId; + } + + public String getName() { + return name; + } + + public String getValue() { + return value; + } + + @Override + public boolean isDisplay() { + return true; + } + + public void setId(long id) { + this.id = id; + } + + public void setVmId(long resourceId) { + this.vmId = resourceId; + } + + public void setName(String name) { + this.name = name; + } + + public void setValue(String value) { + this.value = value; + } +} diff --git a/engine/schema/src/main/java/com/cloud/deployasis/UserVmDeployAsIsDetailVO.java~Stashed changes b/engine/schema/src/main/java/com/cloud/deployasis/UserVmDeployAsIsDetailVO.java~Stashed changes new file mode 100644 index 00000000000..b56b4a49464 --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/deployasis/UserVmDeployAsIsDetailVO.java~Stashed changes @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.deployasis; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Lob; +import javax.persistence.Table; + +import org.apache.cloudstack.api.ResourceDetail; + +@Entity +@Table(name = "user_vm_deploy_as_is_details") +public class UserVmDeployAsIsDetailVO implements ResourceDetail { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + private long id; + + @Column(name = "vm_id") + private long resourceId; + + @Column(name = "name") + private String name; + + @Lob + @Column(name = "value", length = 65535) + private String value; + + public UserVmDeployAsIsDetailVO() { + } + + public UserVmDeployAsIsDetailVO(long vmId, String name, String value) { + this.resourceId = vmId; + this.name = name; + this.value = value; + } + + @Override + public long getId() { + return id; + } + + public long getResourceId() { + return resourceId; + } + + public String getName() { + return name; + } + + public String getValue() { + return value; + } + + @Override + public boolean isDisplay() { + return true; + } + + public void setId(long id) { + this.id = id; + } + + public void setResourceId(long resourceId) { + this.resourceId = resourceId; + } + + public void setName(String name) { + this.name = name; + } + + public void setValue(String value) { + this.value = value; + } +} diff --git a/engine/schema/src/main/java/com/cloud/deployasis/dao/TemplateDeployAsIsDetailsDao.java b/engine/schema/src/main/java/com/cloud/deployasis/dao/TemplateDeployAsIsDetailsDao.java new file mode 100644 index 00000000000..ebbc1eabeb1 --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/deployasis/dao/TemplateDeployAsIsDetailsDao.java @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.deployasis.dao; + +import com.cloud.agent.api.to.deployasis.OVFPropertyTO; +import com.cloud.deployasis.TemplateDeployAsIsDetailVO; +import com.cloud.utils.db.GenericDao; +import com.cloud.agent.api.to.deployasis.OVFNetworkTO; +import org.apache.cloudstack.resourcedetail.ResourceDetailsDao; + +import java.util.List; + +public interface TemplateDeployAsIsDetailsDao extends GenericDao, ResourceDetailsDao { + + OVFPropertyTO findPropertyByTemplateAndKey(long templateId, String key); + List listDetailsByTemplateIdMatchingPrefix(long templateId, String prefix); + List listNetworkRequirementsByTemplateId(long templateId); +} diff --git a/engine/schema/src/main/java/com/cloud/deployasis/dao/TemplateDeployAsIsDetailsDaoImpl.java b/engine/schema/src/main/java/com/cloud/deployasis/dao/TemplateDeployAsIsDetailsDaoImpl.java new file mode 100644 index 00000000000..df21d4b4872 --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/deployasis/dao/TemplateDeployAsIsDetailsDaoImpl.java @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.deployasis.dao; + +import com.cloud.agent.api.to.deployasis.OVFPropertyTO; +import com.cloud.deployasis.DeployAsIsConstants; +import com.cloud.deployasis.TemplateDeployAsIsDetailVO; +import com.cloud.utils.db.SearchCriteria; +import com.google.gson.Gson; +import com.cloud.agent.api.to.deployasis.OVFNetworkTO; +import org.apache.cloudstack.resourcedetail.ResourceDetailsDaoBase; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; + +@Component +public class TemplateDeployAsIsDetailsDaoImpl extends ResourceDetailsDaoBase implements TemplateDeployAsIsDetailsDao { + + private Gson gson = new Gson(); + + public TemplateDeployAsIsDetailsDaoImpl() { + } + + @Override + public void addDetail(long resourceId, String key, String value, boolean display) { + super.addDetail(new TemplateDeployAsIsDetailVO(resourceId, key, value)); + } + + @Override + public OVFPropertyTO findPropertyByTemplateAndKey(long templateId, String key) { + SearchCriteria sc = createSearchCriteria(); + sc.addAnd("templateId", SearchCriteria.Op.EQ, templateId); + sc.addAnd("name", SearchCriteria.Op.EQ, key.startsWith(DeployAsIsConstants.ACS_PROPERTY_PREFIX) ? key : DeployAsIsConstants.ACS_PROPERTY_PREFIX + key); + OVFPropertyTO property = null; + TemplateDeployAsIsDetailVO detail = findOneBy(sc); + if (detail != null) { + property = gson.fromJson(detail.getValue(), OVFPropertyTO.class); + } + return property; + } + + @Override + public List listDetailsByTemplateIdMatchingPrefix(long templateId, String prefix) { + SearchCriteria ssc = createSearchCriteria(); + ssc.addAnd("templateId", SearchCriteria.Op.EQ, templateId); + ssc.addAnd("name", SearchCriteria.Op.LIKE, prefix + "%"); + + return search(ssc, null); + } + + @Override + public List listNetworkRequirementsByTemplateId(long templateId) { + List networkDetails = listDetailsByTemplateIdMatchingPrefix(templateId, DeployAsIsConstants.REQUIRED_NETWORK_PREFIX); + List networkPrereqs = new ArrayList<>(); + for (TemplateDeployAsIsDetailVO property : networkDetails) { + OVFNetworkTO ovfPropertyTO = gson.fromJson(property.getValue(), OVFNetworkTO.class); + networkPrereqs.add(ovfPropertyTO); + } + networkPrereqs.sort(new Comparator() { + @Override + public int compare(OVFNetworkTO o1, OVFNetworkTO o2) { + return o1.getInstanceID() - o2.getInstanceID(); + } + }); + return networkPrereqs; + } +} diff --git a/engine/schema/src/main/java/com/cloud/deployasis/dao/TemplateDeployAsIsDetailsDaoImpl.java~Stashed changes b/engine/schema/src/main/java/com/cloud/deployasis/dao/TemplateDeployAsIsDetailsDaoImpl.java~Stashed changes new file mode 100644 index 00000000000..6f4ec01d1f4 --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/deployasis/dao/TemplateDeployAsIsDetailsDaoImpl.java~Stashed changes @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.deployasis.dao; + +import com.cloud.agent.api.to.deployasis.OVFPropertyTO; +import com.cloud.deployasis.DeployAsIsConstants; +import com.cloud.deployasis.TemplateDeployAsIsDetailVO; +import com.cloud.utils.db.SearchCriteria; +import com.google.gson.Gson; +import com.cloud.agent.api.to.deployasis.OVFNetworkTO; +import org.apache.cloudstack.resourcedetail.ResourceDetailsDaoBase; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; + +@Component +public class TemplateDeployAsIsDetailsDaoImpl extends ResourceDetailsDaoBase implements TemplateDeployAsIsDetailsDao { + + private Gson gson = new Gson(); + + public TemplateDeployAsIsDetailsDaoImpl() { + } + + @Override + public void addDetail(long resourceId, String key, String value, boolean display) { + super.addDetail(new TemplateDeployAsIsDetailVO(resourceId, key, value)); + } + + @Override + public OVFPropertyTO findPropertyByTemplateAndKey(long templateId, String key) { + SearchCriteria sc = createSearchCriteria(); + sc.addAnd("resourceId", SearchCriteria.Op.EQ, templateId); + sc.addAnd("name", SearchCriteria.Op.EQ, key.startsWith(DeployAsIsConstants.ACS_PROPERTY_PREFIX) ? key : DeployAsIsConstants.ACS_PROPERTY_PREFIX + key); + OVFPropertyTO property = null; + TemplateDeployAsIsDetailVO detail = findOneBy(sc); + if (detail != null) { + property = gson.fromJson(detail.getValue(), OVFPropertyTO.class); + } + return property; + } + + @Override + public List listDetailsByTemplateIdMatchingPrefix(long templateId, String prefix) { + SearchCriteria ssc = createSearchCriteria(); + ssc.addAnd("resouceId", SearchCriteria.Op.EQ, templateId); + ssc.addAnd("name", SearchCriteria.Op.LIKE, prefix + "%"); + + return search(ssc, null); + } + + @Override + public List listNetworkRequirementsByTemplateId(long templateId) { + List networkDetails = listDetailsByTemplateIdMatchingPrefix(templateId, DeployAsIsConstants.REQUIRED_NETWORK_PREFIX); + List networkPrereqs = new ArrayList<>(); + for (TemplateDeployAsIsDetailVO property : networkDetails) { + OVFNetworkTO ovfPropertyTO = gson.fromJson(property.getValue(), OVFNetworkTO.class); + networkPrereqs.add(ovfPropertyTO); + } + networkPrereqs.sort(new Comparator() { + @Override + public int compare(OVFNetworkTO o1, OVFNetworkTO o2) { + return o1.getInstanceID() - o2.getInstanceID(); + } + }); + return networkPrereqs; + } +} diff --git a/engine/schema/src/main/java/com/cloud/deployasis/dao/UserVmDeployAsIsDetailsDao.java b/engine/schema/src/main/java/com/cloud/deployasis/dao/UserVmDeployAsIsDetailsDao.java new file mode 100644 index 00000000000..4366e464c1e --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/deployasis/dao/UserVmDeployAsIsDetailsDao.java @@ -0,0 +1,24 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.deployasis.dao; + +import com.cloud.deployasis.UserVmDeployAsIsDetailVO; +import com.cloud.utils.db.GenericDao; +import org.apache.cloudstack.resourcedetail.ResourceDetailsDao; + +public interface UserVmDeployAsIsDetailsDao extends GenericDao, ResourceDetailsDao { +} diff --git a/engine/schema/src/main/java/com/cloud/deployasis/dao/UserVmDeployAsIsDetailsDaoImpl.java b/engine/schema/src/main/java/com/cloud/deployasis/dao/UserVmDeployAsIsDetailsDaoImpl.java new file mode 100644 index 00000000000..8dc5f4a1204 --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/deployasis/dao/UserVmDeployAsIsDetailsDaoImpl.java @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.deployasis.dao; + +import com.cloud.deployasis.UserVmDeployAsIsDetailVO; +import org.apache.cloudstack.resourcedetail.ResourceDetailsDaoBase; +import org.springframework.stereotype.Component; + +@Component +public class UserVmDeployAsIsDetailsDaoImpl extends ResourceDetailsDaoBase implements UserVmDeployAsIsDetailsDao { + + @Override + public void addDetail(long resourceId, String key, String value, boolean display) { + super.addDetail(new UserVmDeployAsIsDetailVO(resourceId, key, value)); + } +} diff --git a/engine/schema/src/main/java/com/cloud/storage/VMTemplateDetailVO.java b/engine/schema/src/main/java/com/cloud/storage/VMTemplateDetailVO.java index 574f4fc500e..5010edfa762 100755 --- a/engine/schema/src/main/java/com/cloud/storage/VMTemplateDetailVO.java +++ b/engine/schema/src/main/java/com/cloud/storage/VMTemplateDetailVO.java @@ -21,7 +21,6 @@ import javax.persistence.Entity; import javax.persistence.GeneratedValue; import javax.persistence.GenerationType; import javax.persistence.Id; -import javax.persistence.Lob; import javax.persistence.Table; import org.apache.cloudstack.api.ResourceDetail; @@ -40,8 +39,7 @@ public class VMTemplateDetailVO implements ResourceDetail { @Column(name = "name") private String name; - @Lob - @Column(name = "value", length = 65535) + @Column(name = "value", length = 1024) private String value; @Column(name = "display") diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDetailsDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDetailsDao.java index 51c9cbbbb40..fe69630ae2e 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDetailsDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDetailsDao.java @@ -16,25 +16,11 @@ // under the License. package com.cloud.storage.dao; -import com.cloud.agent.api.storage.OVFPropertyTO; -import com.cloud.agent.api.to.DatadiskTO; -import org.apache.cloudstack.api.net.NetworkPrerequisiteTO; import org.apache.cloudstack.resourcedetail.ResourceDetailsDao; import com.cloud.storage.VMTemplateDetailVO; import com.cloud.utils.db.GenericDao; -import java.util.List; - public interface VMTemplateDetailsDao extends GenericDao, ResourceDetailsDao { - boolean existsOption(long templateId, String key); - OVFPropertyTO findPropertyByTemplateAndKey(long templateId, String key); - void saveOptions(List opts); - List listPropertiesByTemplateId(long templateId); - List listNetworkRequirementsByTemplateId(long templateId); - List listDisksByTemplateId(long templateId); - - List listDetailsByTemplateIdMatchingPrefix(long templateId, String prefix); - String getTemplateEulaSectionsUrl(long templateId); } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDetailsDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDetailsDaoImpl.java index d904e1a6de7..60e583be3e6 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDetailsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDetailsDaoImpl.java @@ -17,137 +17,17 @@ package com.cloud.storage.dao; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; - -import org.apache.cloudstack.api.net.NetworkPrerequisiteTO; -import org.apache.cloudstack.resourcedetail.ResourceDetailsDaoBase; -import org.apache.commons.collections.CollectionUtils; -import org.apache.log4j.Logger; import org.springframework.stereotype.Component; -import com.cloud.agent.api.storage.OVFPropertyTO; -import com.cloud.agent.api.to.DatadiskTO; -import com.cloud.storage.ImageStore; +import org.apache.cloudstack.resourcedetail.ResourceDetailsDaoBase; + import com.cloud.storage.VMTemplateDetailVO; -import com.cloud.utils.db.SearchBuilder; -import com.cloud.utils.db.SearchCriteria; -import com.cloud.utils.db.TransactionLegacy; -import com.google.gson.Gson; @Component public class VMTemplateDetailsDaoImpl extends ResourceDetailsDaoBase implements VMTemplateDetailsDao { - private final static Logger LOGGER = Logger.getLogger(VMTemplateDetailsDaoImpl.class); - - Gson gson = new Gson(); - - SearchBuilder OptionsSearchBuilder; - - public VMTemplateDetailsDaoImpl() { - super(); - OptionsSearchBuilder = createSearchBuilder(); - OptionsSearchBuilder.and("resourceId", OptionsSearchBuilder.entity().getResourceId(), SearchCriteria.Op.EQ); - OptionsSearchBuilder.and("name", OptionsSearchBuilder.entity().getName(), SearchCriteria.Op.EQ); - OptionsSearchBuilder.done(); - } - @Override public void addDetail(long resourceId, String key, String value, boolean display) { super.addDetail(new VMTemplateDetailVO(resourceId, key, value, display)); } - - @Override - public boolean existsOption(long templateId, String key) { - return findPropertyByTemplateAndKey(templateId, key) != null; - } - - @Override - public OVFPropertyTO findPropertyByTemplateAndKey(long templateId, String key) { - SearchCriteria sc = OptionsSearchBuilder.create(); - sc.setParameters("resourceId", templateId); - sc.setParameters("name", key.startsWith(ImageStore.ACS_PROPERTY_PREFIX) ? key : ImageStore.ACS_PROPERTY_PREFIX + key); - OVFPropertyTO property = null; - VMTemplateDetailVO detail = findOneBy(sc); - if (detail != null) { - property = gson.fromJson(detail.getValue(), OVFPropertyTO.class); - } - return property; - } - - @Override - public void saveOptions(List opts) { - if (CollectionUtils.isEmpty(opts)) { - return; - } - TransactionLegacy txn = TransactionLegacy.currentTxn(); - txn.start(); - for (OVFPropertyTO opt : opts) { - String json = gson.toJson(opt); - VMTemplateDetailVO templateDetailVO = new VMTemplateDetailVO(opt.getTemplateId(), ImageStore.ACS_PROPERTY_PREFIX + opt.getKey(), json, opt.isUserConfigurable()); - persist(templateDetailVO); - } - txn.commit(); - } - - @Override - public List listPropertiesByTemplateId(long templateId) { - List ovfProperties = listDetailsByTemplateIdMatchingPrefix(templateId, ImageStore.ACS_PROPERTY_PREFIX); - List properties = new ArrayList<>(); - for (VMTemplateDetailVO property : ovfProperties) { - OVFPropertyTO ovfPropertyTO = gson.fromJson(property.getValue(), OVFPropertyTO.class); - properties.add(ovfPropertyTO); - } - return properties; - } - - @Override - public List listNetworkRequirementsByTemplateId(long templateId) { - List networkDetails = listDetailsByTemplateIdMatchingPrefix(templateId, ImageStore.REQUIRED_NETWORK_PREFIX); - List networkPrereqs = new ArrayList<>(); - for (VMTemplateDetailVO property : networkDetails) { - NetworkPrerequisiteTO ovfPropertyTO = gson.fromJson(property.getValue(), NetworkPrerequisiteTO.class); - networkPrereqs.add(ovfPropertyTO); - } - networkPrereqs.sort(new Comparator() { - @Override - public int compare(NetworkPrerequisiteTO o1, NetworkPrerequisiteTO o2) { - return o1.getInstanceID() - o2.getInstanceID(); - } - }); - return networkPrereqs; - } - - @Override - public List listDisksByTemplateId(long templateId) { - List diskDefinitions = listDetailsByTemplateIdMatchingPrefix(templateId, ImageStore.DISK_DEFINITION_PREFIX); - List disks = new ArrayList<>(); - for (VMTemplateDetailVO detail : diskDefinitions) { - DatadiskTO datadiskTO = gson.fromJson(detail.getValue(), DatadiskTO.class); - disks.add(datadiskTO); - } - return disks; - } - - @Override - public List listDetailsByTemplateIdMatchingPrefix(long templateId, String prefix) { - SearchCriteria ssc = createSearchCriteria(); - ssc.addAnd("resourceId", SearchCriteria.Op.EQ, templateId); - ssc.addAnd("name", SearchCriteria.Op.LIKE, prefix + "%"); - - return search(ssc, null); - } - - @Override - public String getTemplateEulaSectionsUrl(long templateId) { - List details = findDetails(templateId, ImageStore.OVF_EULA_SECTION_PREFIX); - if (CollectionUtils.isEmpty(details)) { - return null; - } - if (details.size() > 1) { - LOGGER.error("Multiple details for EULA sections for template " + templateId + " returning one"); - } - return details.get(0).getValue(); - } } \ No newline at end of file diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/UserVmDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/UserVmDaoImpl.java index 6bf0dc23168..25479d6658a 100644 --- a/engine/schema/src/main/java/com/cloud/vm/dao/UserVmDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/UserVmDaoImpl.java @@ -30,7 +30,6 @@ import javax.annotation.PostConstruct; import javax.inject.Inject; import com.cloud.offerings.dao.NetworkOfferingServiceMapDao; -import com.cloud.storage.ImageStore; import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; @@ -383,25 +382,13 @@ public class UserVmDaoImpl extends GenericDaoBase implements Use List details = new ArrayList(); for (Map.Entry entry : detailsStr.entrySet()) { - boolean display = visibilityMap.getOrDefault(entry.getKey(), true) && displayOVFDetails(entry.getKey()); + boolean display = visibilityMap.getOrDefault(entry.getKey(), true); details.add(new UserVmDetailVO(vm.getId(), entry.getKey(), entry.getValue(), display)); } _detailsDao.saveDetails(details); } - /* - Do not display VM properties parsed from OVF, handled internally - */ - private boolean displayOVFDetails(String key) { - if (key.startsWith(ImageStore.ACS_PROPERTY_PREFIX) || key.startsWith(ImageStore.OVF_HARDWARE_ITEM_PREFIX) || - key.startsWith(ImageStore.OVF_HARDWARE_CONFIGURATION_PREFIX) || key.startsWith(ImageStore.DISK_DEFINITION_PREFIX) || - key.startsWith(ImageStore.REQUIRED_NETWORK_PREFIX) || key.startsWith(ImageStore.OVF_EULA_SECTION_PREFIX)) { - return false; - } - return true; - } - @Override public List listPodIdsHavingVmsforAccount(long zoneId, long accountId) { TransactionLegacy txn = TransactionLegacy.currentTxn(); diff --git a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml index cde3518e7d2..67b40101d55 100644 --- a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml +++ b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml @@ -295,4 +295,6 @@ + + diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql b/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql index b7e94cc0e86..12c2a8e12ba 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql @@ -473,6 +473,92 @@ ADD CONSTRAINT `fk_template_spool_ref__template_id` ON DELETE NO ACTION ON UPDATE NO ACTION; + +CREATE TABLE IF NOT EXISTS `cloud`.`vsphere_storage_policy` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT, + `uuid` varchar(255) UNIQUE, + `zone_id` bigint(20) unsigned NOT NULL COMMENT 'id of the zone', + `policy_id` varchar(255) NOT NULL COMMENT 'the identifier of the Storage Policy in vSphere DataCenter', + `name` varchar(255) NOT NULL COMMENT 'name of the storage policy', + `description` text COMMENT 'description of the storage policy', + `update_time` datetime COMMENT 'last updated when policy imported', + `removed` datetime COMMENT 'date removed', + PRIMARY KEY (`id`), + KEY `fk_vsphere_storage_policy__zone_id` (`zone_id`), + UNIQUE KEY (`zone_id`, `policy_id`), + CONSTRAINT `fk_vsphere_storage_policy__zone_id` FOREIGN KEY (`zone_id`) REFERENCES `data_center` (`id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +ALTER TABLE `cloud`.`storage_pool` ADD COLUMN `parent` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'ID of the Datastore cluster (storage pool) if this is a child in that Datastore cluster'; + +-- Added parent column to support datastore clusters in vmware vsphere +DROP VIEW IF EXISTS `cloud`.`storage_pool_view`; +CREATE VIEW `cloud`.`storage_pool_view` AS + SELECT + `storage_pool`.`id` AS `id`, + `storage_pool`.`uuid` AS `uuid`, + `storage_pool`.`name` AS `name`, + `storage_pool`.`status` AS `status`, + `storage_pool`.`path` AS `path`, + `storage_pool`.`pool_type` AS `pool_type`, + `storage_pool`.`host_address` AS `host_address`, + `storage_pool`.`created` AS `created`, + `storage_pool`.`removed` AS `removed`, + `storage_pool`.`capacity_bytes` AS `capacity_bytes`, + `storage_pool`.`capacity_iops` AS `capacity_iops`, + `storage_pool`.`scope` AS `scope`, + `storage_pool`.`hypervisor` AS `hypervisor`, + `storage_pool`.`storage_provider_name` AS `storage_provider_name`, + `storage_pool`.`parent` AS `parent`, + `cluster`.`id` AS `cluster_id`, + `cluster`.`uuid` AS `cluster_uuid`, + `cluster`.`name` AS `cluster_name`, + `cluster`.`cluster_type` AS `cluster_type`, + `data_center`.`id` AS `data_center_id`, + `data_center`.`uuid` AS `data_center_uuid`, + `data_center`.`name` AS `data_center_name`, + `data_center`.`networktype` AS `data_center_type`, + `host_pod_ref`.`id` AS `pod_id`, + `host_pod_ref`.`uuid` AS `pod_uuid`, + `host_pod_ref`.`name` AS `pod_name`, + `storage_pool_tags`.`tag` AS `tag`, + `op_host_capacity`.`used_capacity` AS `disk_used_capacity`, + `op_host_capacity`.`reserved_capacity` AS `disk_reserved_capacity`, + `async_job`.`id` AS `job_id`, + `async_job`.`uuid` AS `job_uuid`, + `async_job`.`job_status` AS `job_status`, + `async_job`.`account_id` AS `job_account_id` + FROM + ((((((`storage_pool` + LEFT JOIN `cluster` ON ((`storage_pool`.`cluster_id` = `cluster`.`id`))) + LEFT JOIN `data_center` ON ((`storage_pool`.`data_center_id` = `data_center`.`id`))) + LEFT JOIN `host_pod_ref` ON ((`storage_pool`.`pod_id` = `host_pod_ref`.`id`))) + LEFT JOIN `storage_pool_tags` ON (((`storage_pool_tags`.`pool_id` = `storage_pool`.`id`)))) + LEFT JOIN `op_host_capacity` ON (((`storage_pool`.`id` = `op_host_capacity`.`host_id`) + AND (`op_host_capacity`.`capacity_type` IN (3 , 9))))) + LEFT JOIN `async_job` ON (((`async_job`.`instance_id` = `storage_pool`.`id`) + AND (`async_job`.`instance_type` = 'StoragePool') + AND (`async_job`.`job_status` = 0)))); + + +CREATE TABLE `cloud`.`template_deploy_as_is_details` ( + `id` bigint unsigned NOT NULL auto_increment, + `template_id` bigint unsigned NOT NULL COMMENT 'template id', + `name` varchar(255) NOT NULL, + `value` TEXT, + PRIMARY KEY (`id`), + CONSTRAINT `fk_template_deploy_as_is_details__template_id` FOREIGN KEY `fk_template_deploy_as_is_details__template_id`(`template_id`) REFERENCES `vm_template`(`id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +CREATE TABLE `cloud`.`user_vm_deploy_as_is_details` ( + `id` bigint unsigned NOT NULL auto_increment, + `vm_id` bigint unsigned NOT NULL COMMENT 'virtual machine id', + `name` varchar(255) NOT NULL, + `value` TEXT, + PRIMARY KEY (`id`), + CONSTRAINT `fk_user_vm_deploy_as_is_details__vm_id` FOREIGN KEY `fk_user_vm_deploy_as_is_details__vm_id`(`vm_id`) REFERENCES `vm_instance`(`id`) ON DELETE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + ALTER TABLE `cloud`.`image_store` ADD COLUMN `readonly` boolean DEFAULT false COMMENT 'defines status of image store'; ALTER VIEW `cloud`.`image_store_view` AS diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java index 77d97793405..58c24537d57 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java @@ -18,7 +18,6 @@ */ package org.apache.cloudstack.storage.image; -import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.sql.PreparedStatement; @@ -33,19 +32,8 @@ import java.util.stream.Collectors; import javax.inject.Inject; -import com.cloud.agent.api.storage.OVFConfigurationTO; -import com.cloud.agent.api.storage.OVFEulaSectionTO; -import com.cloud.agent.api.storage.OVFPropertyTO; -import com.cloud.agent.api.storage.OVFVirtualHardwareItemTO; -import com.cloud.agent.api.storage.OVFVirtualHardwareSectionTO; -import com.cloud.storage.ImageStore; import com.cloud.storage.Upload; -import com.cloud.storage.VMTemplateDetailVO; -import com.cloud.utils.compression.CompressionUtil; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import org.apache.cloudstack.api.net.NetworkPrerequisiteTO; -import org.apache.commons.collections.CollectionUtils; +import org.apache.cloudstack.storage.image.deployasis.DeployAsIsHelper; import org.apache.log4j.Logger; import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; @@ -89,7 +77,6 @@ import com.cloud.storage.VMTemplateStorageResourceAssoc; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.VMTemplateDao; -import com.cloud.storage.dao.VMTemplateDetailsDao; import com.cloud.storage.dao.VMTemplateZoneDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.storage.download.DownloadMonitor; @@ -123,14 +110,14 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver { @Inject AlertManager _alertMgr; @Inject - VMTemplateDetailsDao templateDetailsDao; - @Inject DefaultEndPointSelector _defaultEpSelector; @Inject AccountDao _accountDao; @Inject ResourceLimitService _resourceLimitMgr; @Inject + DeployAsIsHelper deployAsIsHelper; + @Inject HostDao hostDao; @Inject CommandExecLogDao _cmdExecLogDao; @@ -143,14 +130,6 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver { protected String _proxy = null; - private static Gson gson; - - static { - GsonBuilder builder = new GsonBuilder(); - builder.disableHtmlEscaping(); - gson = builder.create(); - } - protected Proxy getHttpProxy() { if (_proxy == null) { return null; @@ -211,91 +190,6 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver { } } - /** - * Persist OVF properties as template details for template with id = templateId - */ - private void persistOVFProperties(List ovfProperties, long templateId) { - if (LOGGER.isTraceEnabled()) { - LOGGER.trace(String.format("saving properties for template %d as details", templateId)); - } - for (OVFPropertyTO property : ovfProperties) { - if (LOGGER.isTraceEnabled()) { - LOGGER.trace(String.format("saving property %s for template %d as detail", property.getKey(), templateId)); - } - persistOvfPropertyAsSetOfTemplateDetails(templateId, property); - } - } - - private void persistOvfPropertyAsSetOfTemplateDetails(long templateId, OVFPropertyTO property) { - String key = property.getKey(); - String propKey = ImageStore.ACS_PROPERTY_PREFIX + key; - try { - String propValue = gson.toJson(property); - savePropertyAttribute(templateId, propKey, propValue); - } catch (RuntimeException re) { - LOGGER.error("gson marshalling of property object fails: " + propKey,re); - } - } - - private void persistNetworkRequirements(List networkRequirements, long templateId) { - if (LOGGER.isTraceEnabled()) { - LOGGER.trace(String.format("saving network requirements for template %d as details", templateId)); - } - for (NetworkPrerequisiteTO network : networkRequirements) { - if (LOGGER.isTraceEnabled()) { - LOGGER.trace(String.format("saving property %s for template %d as detail", network.getName(), templateId)); - } - persistRequiredNetworkAsASingleTemplateDetail(templateId, network); - } - } - - private void persistDiskDefinitions(List disks, long templateId) { - if (LOGGER.isTraceEnabled()) { - LOGGER.trace(String.format("saving disk definitionsn for template %d as details", templateId)); - } - for (DatadiskTO disk : disks) { - if (LOGGER.isTraceEnabled()) { - LOGGER.trace(String.format("saving property %s for template %d as detail", disk.getDiskId(), templateId)); - } - persistDiskDefinitionAsASingleTemplateDetail(templateId, disk); - } - } - - private void persistRequiredNetworkAsASingleTemplateDetail(long templateId, NetworkPrerequisiteTO network) { - String key = network.getName(); - String propKey = ImageStore.REQUIRED_NETWORK_PREFIX + key; - try { - String propValue = gson.toJson(network); - savePropertyAttribute(templateId, propKey, propValue); - } catch (RuntimeException re) { - LOGGER.warn("gson marshalling of network object fails: " + propKey,re); - } - } - - private void persistDiskDefinitionAsASingleTemplateDetail(long templateId, DatadiskTO disk) { - String key = disk.getDiskId(); - String propKey = ImageStore.DISK_DEFINITION_PREFIX + key; - try { - String propValue = gson.toJson(disk); - savePropertyAttribute(templateId, propKey, propValue); - } catch (RuntimeException re) { - LOGGER.warn("gson marshalling of disk definition object fails: " + propKey,re); - } - } - - private void savePropertyAttribute(long templateId, String key, String value) { - if ( templateDetailsDao.findDetail(templateId,key) != null) { - LOGGER.debug(String.format("detail '%s' existed for template %d, deleting.", key, templateId)); - templateDetailsDao.removeDetail(templateId,key); - } - if (LOGGER.isTraceEnabled()) { - LOGGER.trace(String.format("template detail for template %d to save is '%s': '%s'", templateId, key, value)); - } - VMTemplateDetailVO detailVO = new VMTemplateDetailVO(templateId, key, value, false); - LOGGER.debug("Persisting template details " + detailVO.getName() + " from OVF properties for template " + templateId); - templateDetailsDao.persist(detailVO); - } - protected Void createTemplateAsyncCallback(AsyncCallbackDispatcher callback, CreateContext context) { if (LOGGER.isDebugEnabled()) { @@ -304,17 +198,14 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver { DownloadAnswer answer = callback.getResult(); DataObject obj = context.data; DataStore store = obj.getDataStore(); - List ovfProperties = answer.getOvfProperties(); - List networkRequirements = answer.getNetworkRequirements(); - List disks = answer.getDisks(); - OVFVirtualHardwareSectionTO ovfHardwareSection = answer.getOvfHardwareSection(); - List eulaSections = answer.getEulaSections(); VMTemplateVO template = _templateDao.findById(obj.getId()); TemplateDataStoreVO tmpltStoreVO = _templateStoreDao.findByStoreTemplate(store.getId(), obj.getId()); if (tmpltStoreVO != null) { if (tmpltStoreVO.getDownloadState() == VMTemplateStorageResourceAssoc.Status.DOWNLOADED) { - persistExtraDetails(obj, ovfProperties, networkRequirements, disks, ovfHardwareSection, eulaSections); + if (template.isDeployAsIs()) { + deployAsIsHelper.persistTemplateDeployAsIsDetails(template.getId(), answer); + } if (LOGGER.isDebugEnabled()) { LOGGER.debug("Template is already in DOWNLOADED state, ignore further incoming DownloadAnswer"); } @@ -354,7 +245,9 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver { templateDaoBuilder.setChecksum(answer.getCheckSum()); _templateDao.update(obj.getId(), templateDaoBuilder); } - persistExtraDetails(obj, ovfProperties, networkRequirements, disks, ovfHardwareSection, eulaSections); + if (template.isDeployAsIs()) { + deployAsIsHelper.persistTemplateDeployAsIsDetails(template.getId(), answer); + } CreateCmdResult result = new CreateCmdResult(null, null); caller.complete(result); @@ -362,76 +255,6 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver { return null; } - private void persistExtraDetails(DataObject obj, List ovfProperties, List networkRequirements, List disks, OVFVirtualHardwareSectionTO ovfHardwareSection, List eulaSections) { - if (LOGGER.isTraceEnabled()) { - LOGGER.trace(String.format("saving %d ovf properties for template '%s' as details", ovfProperties != null ? ovfProperties.size() : 0, obj.getUuid())); - } - if (CollectionUtils.isNotEmpty(ovfProperties)) { - persistOVFProperties(ovfProperties, obj.getId()); - } - if (LOGGER.isTraceEnabled()) { - LOGGER.trace(String.format("saving %d required network requirements for template '%s' as details", networkRequirements != null ? networkRequirements.size() : 0, obj.getUuid())); - } - if (CollectionUtils.isNotEmpty(networkRequirements)) { - persistNetworkRequirements(networkRequirements, obj.getId()); - } - if (LOGGER.isTraceEnabled()) { - LOGGER.trace(String.format("saving %d disks definitions for template '%s' as details", disks != null ? disks.size() : 0, obj.getUuid())); - } - if (CollectionUtils.isNotEmpty(disks)) { - persistDiskDefinitions(disks, obj.getId()); - } - if (CollectionUtils.isNotEmpty(eulaSections)) { - persistEulaSectionsAsTemplateDetails(eulaSections, obj.getId()); - } - persistOVFHardwareSectionAsTemplateDetails(ovfHardwareSection, obj.getId()); - } - - private void persistEulaSectionsAsTemplateDetails(List eulaSections, long templateId) { - CompressionUtil compressionUtil = new CompressionUtil(); - for (OVFEulaSectionTO eulaSectionTO : eulaSections) { - String key = ImageStore.OVF_EULA_SECTION_PREFIX + eulaSectionTO.getIndex() + "-" + eulaSectionTO.getInfo(); - byte[] compressedLicense = eulaSectionTO.getCompressedLicense(); - try { - String detailValue = compressionUtil.decompressByteArary(compressedLicense); - savePropertyAttribute(templateId, key, detailValue); - } catch (IOException e) { - LOGGER.error("Could not decompress the license for template " + templateId, e); - } - } - } - - /** - * Persist template details for template with ID=templateId, with name=key and value=json(object) - */ - private void persistTemplateDetailGsonEncoded(long templateId, String key, Object object) { - try { - String propValue = gson.toJson(object); - savePropertyAttribute(templateId, key, propValue); - } catch (RuntimeException re) { - LOGGER.error("gson marshalling of property object fails: " + key, re); - } - } - - private void persistOVFHardwareSectionAsTemplateDetails(OVFVirtualHardwareSectionTO ovfHardwareSection, long templateId) { - if (ovfHardwareSection != null) { - if (CollectionUtils.isNotEmpty(ovfHardwareSection.getConfigurations())) { - for (OVFConfigurationTO configuration : ovfHardwareSection.getConfigurations()) { - String key = configuration.getId(); - String propKey = ImageStore.OVF_HARDWARE_CONFIGURATION_PREFIX + configuration.getIndex() + "-" + key; - persistTemplateDetailGsonEncoded(templateId, propKey, configuration); - } - } - if (CollectionUtils.isNotEmpty(ovfHardwareSection.getCommonHardwareItems())) { - for (OVFVirtualHardwareItemTO item : ovfHardwareSection.getCommonHardwareItems()) { - String key = item.getResourceType().getName().trim().replaceAll("\\s","") + "-" + item.getInstanceId(); - String propKey = ImageStore.OVF_HARDWARE_ITEM_PREFIX + key; - persistTemplateDetailGsonEncoded(templateId, propKey, item); - } - } - } - } - protected Void createVolumeAsyncCallback(AsyncCallbackDispatcher callback, CreateContext context) { DownloadAnswer answer = callback.getResult(); diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/deployasis/DeployAsIsHelper.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/deployasis/DeployAsIsHelper.java new file mode 100644 index 00000000000..b23326b0406 --- /dev/null +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/deployasis/DeployAsIsHelper.java @@ -0,0 +1,15 @@ +package org.apache.cloudstack.storage.image.deployasis; + +import com.cloud.agent.api.storage.DownloadAnswer; +import com.cloud.vm.VirtualMachineProfile; + +import java.util.Map; + +public interface DeployAsIsHelper { + + void persistTemplateDeployAsIsDetails(long templateId, DownloadAnswer answer); + Map getVirtualMachineDeployAsIsProperties(VirtualMachineProfile vmId); + + String getAllocatedVirtualMachineTemplatePath(VirtualMachineProfile vm, String configuration, String destStoragePool); + String getAllocatedVirtualMachineDestinationStoragePool(VirtualMachineProfile vm); +} diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/deployasis/DeployAsIsHelperImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/deployasis/DeployAsIsHelperImpl.java new file mode 100644 index 00000000000..69982cd798d --- /dev/null +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/deployasis/DeployAsIsHelperImpl.java @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.image.deployasis; + +import com.cloud.agent.api.storage.DownloadAnswer; +import com.cloud.agent.api.to.DataStoreTO; +import com.cloud.agent.api.to.DataTO; +import com.cloud.agent.api.to.DiskTO; +import com.cloud.agent.api.to.deployasis.OVFConfigurationTO; +import com.cloud.agent.api.to.deployasis.OVFEulaSectionTO; +import com.cloud.agent.api.to.deployasis.OVFPropertyTO; +import com.cloud.agent.api.to.deployasis.OVFVirtualHardwareItemTO; +import com.cloud.agent.api.to.deployasis.OVFVirtualHardwareSectionTO; +import com.cloud.agent.api.to.deployasis.TemplateDeployAsIsInformationTO; +import com.cloud.deployasis.DeployAsIsConstants; +import com.cloud.deployasis.TemplateDeployAsIsDetailVO; +import com.cloud.deployasis.UserVmDeployAsIsDetailVO; +import com.cloud.deployasis.dao.TemplateDeployAsIsDetailsDao; +import com.cloud.deployasis.dao.UserVmDeployAsIsDetailsDao; +import com.cloud.storage.VMTemplateStoragePoolVO; +import com.cloud.storage.Volume; +import com.cloud.storage.dao.VMTemplatePoolDao; +import com.cloud.utils.compression.CompressionUtil; +import com.cloud.utils.crypt.DBEncryptionUtil; +import com.cloud.vm.VirtualMachineProfile; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.cloud.agent.api.to.deployasis.OVFNetworkTO; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.commons.collections.CollectionUtils; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Component +public class DeployAsIsHelperImpl implements DeployAsIsHelper { + + private static final Logger LOGGER = Logger.getLogger(DeployAsIsHelperImpl.class); + private static Gson gson; + + @Inject + private TemplateDeployAsIsDetailsDao templateDeployAsIsDetailsDao; + @Inject + private UserVmDeployAsIsDetailsDao userVmDeployAsIsDetailsDao; + @Inject + private PrimaryDataStoreDao storagePoolDao; + @Inject + private VMTemplatePoolDao templateStoragePoolDao; + + static { + GsonBuilder builder = new GsonBuilder(); + builder.disableHtmlEscaping(); + gson = builder.create(); + } + + public void persistTemplateDeployAsIsDetails(long templateId, DownloadAnswer answer) { + List ovfProperties = answer.getOvfProperties(); + List networkRequirements = answer.getNetworkRequirements(); + OVFVirtualHardwareSectionTO ovfHardwareSection = answer.getOvfHardwareSection(); + List eulaSections = answer.getEulaSections(); + + if (CollectionUtils.isNotEmpty(ovfProperties)) { + persistTemplateDeployAsIsInformationTOList(templateId, ovfProperties); + } + if (CollectionUtils.isNotEmpty(networkRequirements)) { + persistTemplateDeployAsIsInformationTOList(templateId, networkRequirements); + } + if (CollectionUtils.isNotEmpty(eulaSections)) { + persistTemplateDeployAsIsInformationTOList(templateId, eulaSections); + } + if (ovfHardwareSection != null) { + if (CollectionUtils.isNotEmpty(ovfHardwareSection.getConfigurations())) { + persistTemplateDeployAsIsInformationTOList(templateId, ovfHardwareSection.getConfigurations()); + } + if (CollectionUtils.isNotEmpty(ovfHardwareSection.getCommonHardwareItems())) { + persistTemplateDeployAsIsInformationTOList(templateId, ovfHardwareSection.getCommonHardwareItems()); + } + } + } + + @Override + public Map getVirtualMachineDeployAsIsProperties(VirtualMachineProfile vm) { + Map map = new HashMap<>(); + List details = userVmDeployAsIsDetailsDao.listDetails(vm.getId()); + if (CollectionUtils.isNotEmpty(details)) { + for (UserVmDeployAsIsDetailVO detail : details) { + OVFPropertyTO property = templateDeployAsIsDetailsDao.findPropertyByTemplateAndKey(vm.getTemplateId(), detail.getName()); + String value = property.isPassword() ? DBEncryptionUtil.decrypt(detail.getValue()) : detail.getValue(); + map.put(detail.getName(), value); + } + } + return map; + } + + @Override + public String getAllocatedVirtualMachineTemplatePath(VirtualMachineProfile vm, String configuration, String destStoragePool) { + StoragePoolVO storagePoolVO = storagePoolDao.findByUuid(destStoragePool); + VMTemplateStoragePoolVO tmplRef = templateStoragePoolDao.findByPoolTemplate(storagePoolVO.getId(), + vm.getTemplate().getId(), configuration); + if (tmplRef != null) { + return tmplRef.getInstallPath(); + } + return null; + } + + @Override + public String getAllocatedVirtualMachineDestinationStoragePool(VirtualMachineProfile vm) { + if (vm != null) { + if (CollectionUtils.isNotEmpty(vm.getDisks())) { + for (DiskTO disk : vm.getDisks()) { + if (disk.getType() == Volume.Type.ISO) { + continue; + } + DataTO data = disk.getData(); + if (data != null) { + DataStoreTO dataStore = data.getDataStore(); + if (dataStore != null) { + return dataStore.getUuid(); + } + } + } + } + } + return null; + } + + private void persistTemplateDeployAsIsInformationTOList(long templateId, + List informationTOList) { + for (TemplateDeployAsIsInformationTO informationTO : informationTOList) { + String propKey = getKeyFromInformationTO(informationTO); + if (LOGGER.isTraceEnabled()) { + LOGGER.trace(String.format("Saving property %s for template %d as detail", propKey, templateId)); + } + String propValue = null; + try { + propValue = getValueFromInformationTO(informationTO); + } catch (RuntimeException re) { + LOGGER.error("gson marshalling of property object fails: " + propKey,re); + } catch (IOException e) { + LOGGER.error("Could not decompress the license for template " + templateId, e); + } + saveTemplateDeployAsIsPropertyAttribute(templateId, propKey, propValue); + } + } + + private String getValueFromInformationTO(TemplateDeployAsIsInformationTO informationTO) throws IOException { + if (informationTO instanceof OVFEulaSectionTO) { + CompressionUtil compressionUtil = new CompressionUtil(); + byte[] compressedLicense = ((OVFEulaSectionTO) informationTO).getCompressedLicense(); + return compressionUtil.decompressByteArary(compressedLicense); + } + return gson.toJson(informationTO); + } + + private String getKeyFromInformationTO(TemplateDeployAsIsInformationTO informationTO) { + if (informationTO instanceof OVFPropertyTO) { + return DeployAsIsConstants.ACS_PROPERTY_PREFIX + ((OVFPropertyTO) informationTO).getKey(); + } else if (informationTO instanceof OVFNetworkTO) { + return DeployAsIsConstants.REQUIRED_NETWORK_PREFIX + ((OVFNetworkTO) informationTO).getName(); + } else if (informationTO instanceof OVFConfigurationTO) { + return DeployAsIsConstants.OVF_HARDWARE_CONFIGURATION_PREFIX + + ((OVFConfigurationTO) informationTO).getIndex() + "-" + ((OVFConfigurationTO) informationTO).getId(); + } else if (informationTO instanceof OVFVirtualHardwareItemTO) { + String key = ((OVFVirtualHardwareItemTO) informationTO).getResourceType().getName().trim().replaceAll("\\s","") + + "-" + ((OVFVirtualHardwareItemTO) informationTO).getInstanceId(); + return DeployAsIsConstants.OVF_HARDWARE_ITEM_PREFIX + key; + } else if (informationTO instanceof OVFEulaSectionTO) { + return DeployAsIsConstants.OVF_EULA_SECTION_PREFIX + ((OVFEulaSectionTO) informationTO).getIndex() + + "-" + ((OVFEulaSectionTO) informationTO).getInfo(); + } + return null; + } + + private void saveTemplateDeployAsIsPropertyAttribute(long templateId, String key, String value) { + if (LOGGER.isTraceEnabled()) { + LOGGER.trace(String.format("Saving property %s for template %d as detail", key, templateId)); + } + if (templateDeployAsIsDetailsDao.findDetail(templateId,key) != null) { + LOGGER.debug(String.format("Detail '%s' existed for template %d, deleting.", key, templateId)); + templateDeployAsIsDetailsDao.removeDetail(templateId,key); + } + if (LOGGER.isTraceEnabled()) { + LOGGER.trace(String.format("Template detail for template %d to save is '%s': '%s'", templateId, key, value)); + } + TemplateDeployAsIsDetailVO detailVO = new TemplateDeployAsIsDetailVO(templateId, key, value); + LOGGER.debug("Persisting template details " + detailVO.getName() + " from OVF properties for template " + templateId); + templateDeployAsIsDetailsDao.persist(detailVO); + } + +} diff --git a/engine/storage/src/main/resources/META-INF/cloudstack/core/spring-engine-storage-core-context.xml b/engine/storage/src/main/resources/META-INF/cloudstack/core/spring-engine-storage-core-context.xml index 33385b5ae00..5cecb224125 100644 --- a/engine/storage/src/main/resources/META-INF/cloudstack/core/spring-engine-storage-core-context.xml +++ b/engine/storage/src/main/resources/META-INF/cloudstack/core/spring-engine-storage-core-context.xml @@ -72,4 +72,6 @@ + + diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java index 7ffc88af531..56a875d8ab2 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java @@ -16,9 +16,7 @@ // under the License. package com.cloud.hypervisor.guru; -import com.cloud.agent.api.storage.OVFPropertyTO; import com.cloud.agent.api.to.DeployAsIsInfoTO; -import com.cloud.agent.api.to.DiskTO; import com.cloud.agent.api.to.NicTO; import com.cloud.agent.api.to.VirtualMachineTO; import com.cloud.exception.InsufficientAddressCapacityException; @@ -35,12 +33,8 @@ import com.cloud.network.dao.NetworkDao; import com.cloud.network.dao.NetworkVO; import com.cloud.storage.GuestOSHypervisorVO; import com.cloud.storage.GuestOSVO; -import com.cloud.storage.ImageStore; -import com.cloud.storage.VMTemplateStoragePoolVO; -import com.cloud.storage.Volume; import com.cloud.storage.dao.GuestOSDao; import com.cloud.storage.dao.GuestOSHypervisorDao; -import com.cloud.storage.dao.VMTemplateDetailsDao; import com.cloud.storage.dao.VMTemplatePoolDao; import com.cloud.template.VirtualMachineTemplate; import com.cloud.utils.exception.CloudRuntimeException; @@ -51,10 +45,8 @@ import com.cloud.vm.VirtualMachineProfile; import com.cloud.vm.VmDetailConstants; import com.cloud.vm.dao.DomainRouterDao; import com.cloud.vm.dao.NicDao; -import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; -import org.apache.commons.collections.CollectionUtils; +import org.apache.cloudstack.storage.image.deployasis.DeployAsIsHelper; import org.apache.commons.lang.BooleanUtils; import org.apache.log4j.Logger; @@ -65,7 +57,6 @@ import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; class VmwareVmImplementer { private static final Logger LOGGER = Logger.getLogger(VmwareVmImplementer.class); @@ -89,9 +80,9 @@ class VmwareVmImplementer { @Inject VMTemplatePoolDao templateStoragePoolDao; @Inject - VMTemplateDetailsDao templateDetailsDao; - @Inject VmwareManager vmwareMgr; + @Inject + DeployAsIsHelper deployAsIsHelper; private Boolean globalNestedVirtualisationEnabled; private Boolean globalNestedVPerVMEnabled; @@ -183,9 +174,7 @@ class VmwareVmImplementer { } if (deployAsIs) { - List ovfProperties = getOvfPropertyList(vm, details); - handleOvfProperties(vm, to, details, ovfProperties); - setDeployAsIsParams(vm, to, details); + setDeployAsIsInfoTO(vm, to, details); } setDetails(to, details); @@ -193,44 +182,25 @@ class VmwareVmImplementer { return to; } - private void setDeployAsIsParams(VirtualMachineProfile vm, VirtualMachineTO to, Map details) { - DeployAsIsInfoTO info = new DeployAsIsInfoTO(); - - String configuration = null; - if (details.containsKey(VmDetailConstants.DEPLOY_AS_IS_CONFIGURATION)) { - configuration = details.get(VmDetailConstants.DEPLOY_AS_IS_CONFIGURATION); - info.setDeploymentConfiguration(configuration); - } - - // Deploy as-is disks are all allocated to the same storage pool - String deployAsIsStoreUuid = vm.getDisks().get(0).getData().getDataStore().getUuid(); - StoragePoolVO storagePoolVO = storagePoolDao.findByUuid(deployAsIsStoreUuid); - VMTemplateStoragePoolVO tmplRef = templateStoragePoolDao.findByPoolTemplate(storagePoolVO.getId(), vm.getTemplate().getId(), configuration); - if (tmplRef != null) { - info.setTemplatePath(tmplRef.getInstallPath()); - } - - info.setDeployAsIs(true); + /** + * Set the information relevant for deploy-as-is VMs on the VM TO + */ + private void setDeployAsIsInfoTO(VirtualMachineProfile vm, VirtualMachineTO to, Map details) { + String configuration = details.getOrDefault(VmDetailConstants.DEPLOY_AS_IS_CONFIGURATION, null); + Map properties = deployAsIsHelper.getVirtualMachineDeployAsIsProperties(vm); + String destStoragePool = deployAsIsHelper.getAllocatedVirtualMachineDestinationStoragePool(vm); + String templatePath = deployAsIsHelper.getAllocatedVirtualMachineTemplatePath(vm, configuration, destStoragePool); + DeployAsIsInfoTO info = new DeployAsIsInfoTO(templatePath, destStoragePool, properties); to.setDeployAsIsInfo(info); } private void setDetails(VirtualMachineTO to, Map details) { - Map detailsToSend = new HashMap<>(); - for (String key: details.keySet()) { - if (key.startsWith(ImageStore.OVF_EULA_SECTION_PREFIX) || - key.startsWith(ImageStore.OVF_HARDWARE_CONFIGURATION_PREFIX) || - key.startsWith(ImageStore.OVF_HARDWARE_ITEM_PREFIX)) { - if (LOGGER.isTraceEnabled()) { - LOGGER.trace(String.format("Discarding detail for VM %s: %s => %s", to.getName(), key, details.get(key))); - } - continue; - } - if (LOGGER.isTraceEnabled()) { + if (LOGGER.isTraceEnabled()) { + for (String key : details.keySet()) { LOGGER.trace(String.format("Detail for VM %s: %s => %s", to.getName(), key, details.get(key))); } - detailsToSend.put(key, details.get(key)); } - to.setDetails(detailsToSend); + to.setDetails(details); } private void configureDomainRouterNicsAndDetails(VirtualMachineProfile vm, VirtualMachineTO to, Map details, List nicProfiles) { @@ -318,49 +288,6 @@ class VmwareVmImplementer { } } - private void handleOvfProperties(VirtualMachineProfile vm, VirtualMachineTO to, Map details, List ovfProperties) { - if (CollectionUtils.isNotEmpty(ovfProperties)) { - removeOvfPropertiesFromDetails(ovfProperties, details); - to.setOvfProperties(ovfProperties); - } - } - - private DiskTO getRootDiskTOFromVM(VirtualMachineProfile vm) { - DiskTO rootDiskTO; - List rootDiskList; - rootDiskList = vm.getDisks().stream().filter(x -> x.getType() == Volume.Type.ROOT).collect(Collectors.toList()); - if (rootDiskList.size() != 1) { - if (vm.getTemplate().isDeployAsIs()) { - rootDiskList = vm.getDisks().stream().filter(x -> x.getType() == null).collect(Collectors.toList()); - if (rootDiskList.size() < 1) { - throw new CloudRuntimeException("Did not find a template to serve as root disk for VM " + vm.getHostName()); - } - } else { - throw new CloudRuntimeException("Did not find only one root disk for VM " + vm.getHostName()); - } - } - rootDiskTO = rootDiskList.get(0); - return rootDiskTO; - } - - private List getOvfPropertyList(VirtualMachineProfile vm, Map details) { - List ovfProperties = new ArrayList(); - for (String detailKey : details.keySet()) { - if (detailKey.startsWith(ImageStore.ACS_PROPERTY_PREFIX)) { - OVFPropertyTO propertyTO = templateDetailsDao.findPropertyByTemplateAndKey(vm.getTemplateId(), detailKey); - String vmPropertyKey = detailKey.replace(ImageStore.ACS_PROPERTY_PREFIX, ""); - if (propertyTO == null) { - LOGGER.warn(String.format("OVF property %s not found on template, discarding", vmPropertyKey)); - continue; - } - propertyTO.setKey(vmPropertyKey); - propertyTO.setValue(details.get(detailKey)); - ovfProperties.add(propertyTO); - } - } - return ovfProperties; - } - private void addReservationDetails(long clusterId, Map details) { details.put(VMwareGuru.VmwareReserveCpu.key(), VMwareGuru.VmwareReserveCpu.valueIn(clusterId).toString()); details.put(VMwareGuru.VmwareReserveMemory.key(), VMwareGuru.VmwareReserveMemory.valueIn(clusterId).toString()); @@ -408,16 +335,6 @@ class VmwareVmImplementer { // details.put(VmDetailConstants.BOOT_TYPE, to.getBootType()); } - /* - Remove OVF properties from details to be sent to hypervisor (avoid duplicate data) - */ - private void removeOvfPropertiesFromDetails(List ovfProperties, Map details) { - for (OVFPropertyTO propertyTO : ovfProperties) { - String key = propertyTO.getKey(); - details.remove(ApiConstants.PROPERTIES + "-" + key); - } - } - /** * Adds {@code 'nestedVirtualizationFlag'} value to {@code details} due to if it should be enabled or not * @param details vm details should not be null diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 810a4d38117..f4d2c6d424c 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -47,7 +47,6 @@ import javax.xml.datatype.XMLGregorianCalendar; import com.cloud.agent.api.to.DataTO; import com.cloud.agent.api.to.DeployAsIsInfoTO; -import com.cloud.storage.ImageStore; import com.cloud.agent.api.ValidateVcenterDetailsCommand; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.storage.configdrive.ConfigDrive; @@ -56,7 +55,6 @@ import org.apache.cloudstack.storage.to.VolumeObjectTO; import org.apache.cloudstack.utils.volume.VirtualMachineDiskInfo; import org.apache.cloudstack.vm.UnmanagedInstanceTO; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.math.NumberUtils; @@ -180,7 +178,7 @@ import com.cloud.agent.api.storage.CreatePrivateTemplateAnswer; import com.cloud.agent.api.storage.DestroyCommand; import com.cloud.agent.api.storage.MigrateVolumeAnswer; import com.cloud.agent.api.storage.MigrateVolumeCommand; -import com.cloud.agent.api.storage.OVFPropertyTO; +import com.cloud.agent.api.to.deployasis.OVFPropertyTO; import com.cloud.agent.api.storage.PrimaryStorageDownloadAnswer; import com.cloud.agent.api.storage.PrimaryStorageDownloadCommand; import com.cloud.agent.api.storage.ResizeVolumeAnswer; @@ -253,7 +251,6 @@ import com.cloud.utils.ExecutionResult; import com.cloud.utils.NumbersUtil; import com.cloud.utils.Pair; import com.cloud.utils.Ternary; -import com.cloud.utils.crypt.DBEncryptionUtil; import com.cloud.utils.db.DB; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.exception.ExceptionUtil; @@ -1842,14 +1839,13 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa DiskTO[] specDisks = vmSpec.getDisks(); DeployAsIsInfoTO deployAsIsInfo = vmSpec.getDeployAsIsInfo(); - boolean installAsIs = deployAsIsInfo != null && deployAsIsInfo.isDeployAsIs(); + boolean installAsIs = deployAsIsInfo != null; if (installAsIs && dcMo.findVm(vmInternalCSName) == null) { if (s_logger.isTraceEnabled()) { s_logger.trace("Deploying OVA from as is"); } String deployAsIsTemplate = deployAsIsInfo.getTemplatePath(); - String destDatastore = getDatastoreFromSpecDisks(specDisks); - String deploymentConfiguration = deployAsIsInfo.getDeploymentConfiguration(); + String destDatastore = deployAsIsInfo.getDestStoragePool(); vmInVcenter = _storageProcessor.cloneVMFromTemplate(deployAsIsTemplate, vmInternalCSName, destDatastore); mapSpecDisksToClonedDisks(vmInVcenter, vmInternalCSName, specDisks); } @@ -1996,7 +1992,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa if (s_logger.isTraceEnabled()) { s_logger.trace(String.format("current count(s) desired: %d/ found:%d. now adding device to device count for vApp config ISO", totalChangeDevices, hackDeviceCount)); } - if (vmSpec.getOvfProperties() != null) { + if (deployAsIsInfo != null && deployAsIsInfo.getProperties() != null) { totalChangeDevices++; } @@ -2381,18 +2377,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa // config video card configureVideoCard(vmMo, vmSpec, vmConfigSpec); - // Set OVF properties (if available) - List ovfProperties = vmSpec.getOvfProperties(); - VmConfigInfo templateVappConfig; - if (ovfProperties != null) { - VirtualMachineMO templateVMmo = dcMo.findVm(deployAsIsInfo.getTemplatePath()); - templateVappConfig = templateVMmo.getConfigInfo().getVAppConfig(); - // Set OVF properties (if available) - if (CollectionUtils.isNotEmpty(ovfProperties)) { - s_logger.info("Copying OVF properties from template and setting them to the values the user provided"); - copyVAppConfigsFromTemplate(templateVappConfig, ovfProperties, vmConfigSpec); - } - } + setDeployAsIsProperties(vmMo, deployAsIsInfo, vmConfigSpec); setBootOptions(vmSpec, bootMode, vmConfigSpec); @@ -2483,26 +2468,17 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } } - private String getDatastoreFromSpecDisks(DiskTO[] specDisks) { - if (specDisks.length == 0) { - return null; + /** + * Set OVF properties (if available) + */ + private void setDeployAsIsProperties(VirtualMachineMO vmMo, DeployAsIsInfoTO deployAsIsInfo, + VirtualMachineConfigSpec vmConfigSpec) throws Exception { + if (deployAsIsInfo != null) { + Map properties = deployAsIsInfo.getProperties(); + VmConfigInfo vAppConfig = vmMo.getConfigInfo().getVAppConfig(); + s_logger.info("Copying OVF properties to the values the user provided"); + setVAppPropertiesToConfigSpec(vAppConfig, properties, vmConfigSpec); } - - Map> psDisksMap = Arrays.asList(specDisks).stream() - .filter(x -> x.getType() != Volume.Type.ISO && x.getData() != null && x.getData().getDataStore() != null) - .collect(Collectors.groupingBy(x -> x.getData().getDataStore().getUuid())); - - String dataStore; - if (MapUtils.isEmpty(psDisksMap)) { - s_logger.error("Could not find a destination datastore for VM volumes"); - return null; - } else { - dataStore = psDisksMap.keySet().iterator().next(); - if (psDisksMap.keySet().size() > 1) { - s_logger.info("Found multiple destination datastores for VM volumes, selecting " + dataStore); - } - } - return dataStore; } /** @@ -2624,17 +2600,14 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa /** * Set the properties section from existing vApp configuration and values set on ovfProperties */ - protected List copyVAppConfigPropertySectionFromOVF(VmConfigInfo vAppConfig, List ovfProperties) { + protected List copyVAppConfigPropertySectionFromOVF(VmConfigInfo vAppConfig, Map ovfProperties) { List productFromOvf = vAppConfig.getProperty(); List specs = new ArrayList<>(); - Map> ovfMap = getOVFMap(ovfProperties); for (VAppPropertyInfo info : productFromOvf) { VAppPropertySpec spec = new VAppPropertySpec(); - if (ovfMap.containsKey(info.getId())) { - Pair pair = ovfMap.get(info.getId()); - String value = pair.first(); - boolean isPassword = pair.second(); - info.setValue(isPassword ? DBEncryptionUtil.decrypt(value) : value); + if (ovfProperties.containsKey(info.getId())) { + String value = ovfProperties.get(info.getId()); + info.setValue(value); } spec.setInfo(info); spec.setOperation(ArrayUpdateOperation.ADD); @@ -2662,9 +2635,9 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa * Set the vApp configuration to vmConfig spec, copying existing configuration from vAppConfig * and seting properties values from ovfProperties */ - protected void copyVAppConfigsFromTemplate(VmConfigInfo vAppConfig, - List ovfProperties, - VirtualMachineConfigSpec vmConfig) throws Exception { + protected void setVAppPropertiesToConfigSpec(VmConfigInfo vAppConfig, + Map ovfProperties, + VirtualMachineConfigSpec vmConfig) throws Exception { VmConfigSpec vmConfigSpec = new VmConfigSpec(); vmConfigSpec.getEula().addAll(vAppConfig.getEula()); vmConfigSpec.setInstallBootStopDelay(vAppConfig.getInstallBootStopDelay()); @@ -3009,10 +2982,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa private static void configCustomExtraOption(List extraOptions, VirtualMachineTO vmSpec) { // we no longer to validation anymore for (Map.Entry entry : vmSpec.getDetails().entrySet()) { - if (entry.getKey().equalsIgnoreCase(VmDetailConstants.BOOT_MODE) || - entry.getKey().startsWith(ImageStore.REQUIRED_NETWORK_PREFIX) || - entry.getKey().startsWith(ImageStore.ACS_PROPERTY_PREFIX) || - entry.getKey().startsWith(ImageStore.DISK_DEFINITION_PREFIX)) { + if (entry.getKey().equalsIgnoreCase(VmDetailConstants.BOOT_MODE)) { continue; } OptionValue newVal = new OptionValue(); diff --git a/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDaoImpl.java index e51dd1e3f9c..9cfe710fd71 100644 --- a/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDaoImpl.java @@ -26,6 +26,9 @@ import java.util.Set; import javax.inject.Inject; +import com.cloud.deployasis.DeployAsIsConstants; +import com.cloud.deployasis.TemplateDeployAsIsDetailVO; +import com.cloud.deployasis.dao.TemplateDeployAsIsDetailsDao; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.utils.security.DigestHelper; import org.apache.log4j.Logger; @@ -83,6 +86,8 @@ public class TemplateJoinDaoImpl extends GenericDaoBaseWithTagInformation tmpltIdPairSearch; @@ -240,6 +245,8 @@ public class TemplateJoinDaoImpl extends GenericDaoBaseWithTagInformation details = _templateDetailsDao.listDetailsKeyPairs(template.getId()); templateResponse.setDetails(details); + + setDeployAsIsDetails(template, templateResponse); } // update tag information @@ -272,6 +279,19 @@ public class TemplateJoinDaoImpl extends GenericDaoBaseWithTagInformation deployAsIsDetails = templateDeployAsIsDetailsDao.listDetails(template.getId()); + for (TemplateDeployAsIsDetailVO deployAsIsDetailVO : deployAsIsDetails) { + if (deployAsIsDetailVO.getName().startsWith(DeployAsIsConstants.OVF_HARDWARE_ITEM_PREFIX)) { + //Do not list hardware items + continue; + } + templateResponse.addDeployAsIsDetail(deployAsIsDetailVO.getName(), deployAsIsDetailVO.getValue()); + } + } + } + //TODO: This is to keep compatibility with 4.1 API, where updateTemplateCmd and updateIsoCmd will return a simpler TemplateResponse // compared to listTemplates and listIsos. @Override diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index a7f01cb1be3..8395467039d 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -47,12 +47,12 @@ import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; -import com.cloud.agent.api.storage.OVFPropertyTO; +import com.cloud.agent.api.to.deployasis.OVFPropertyTO; +import com.cloud.deployasis.UserVmDeployAsIsDetailVO; +import com.cloud.deployasis.dao.UserVmDeployAsIsDetailsDao; import com.cloud.exception.UnsupportedServiceException; import com.cloud.hypervisor.Hypervisor; -import com.cloud.storage.ImageStore; -import com.cloud.storage.VMTemplateDetailVO; -import com.cloud.storage.dao.VMTemplateDetailsDao; +import com.cloud.deployasis.dao.TemplateDeployAsIsDetailsDao; import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.acl.SecurityChecker.AccessType; import org.apache.cloudstack.affinity.AffinityGroupService; @@ -83,7 +83,7 @@ import org.apache.cloudstack.api.command.user.vm.UpgradeVMCmd; import org.apache.cloudstack.api.command.user.vmgroup.CreateVMGroupCmd; import org.apache.cloudstack.api.command.user.vmgroup.DeleteVMGroupCmd; import org.apache.cloudstack.api.command.user.volume.ResizeVolumeCmd; -import org.apache.cloudstack.api.net.NetworkPrerequisiteTO; +import com.cloud.agent.api.to.deployasis.OVFNetworkTO; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.cloud.entity.api.VirtualMachineEntity; import org.apache.cloudstack.engine.cloud.entity.api.db.dao.VMNetworkMapDao; @@ -507,7 +507,9 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir @Inject private ResourceTagDao resourceTagDao; @Inject - private VMTemplateDetailsDao templateDetailsDao; + private TemplateDeployAsIsDetailsDao templateDeployAsIsDetailsDao; + @Inject + private UserVmDeployAsIsDetailsDao userVmDeployAsIsDetailsDao; private ScheduledExecutorService _executor = null; private ScheduledExecutorService _vmIpFetchExecutor = null; @@ -2503,17 +2505,6 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir } } } - for (String detailName : details.keySet()) { - if (s_logger.isTraceEnabled()) { - s_logger.trace(String.format("looking for vm detail '%s'", detailName)); - } - if (detailName.startsWith(ImageStore.ACS_PROPERTY_PREFIX)) { - OVFPropertyTO propertyTO = templateDetailsDao.findPropertyByTemplateAndKey(vmInstance.getTemplateId(),detailName); - if (propertyTO != null && propertyTO.isPassword()) { - details.put(detailName, DBEncryptionUtil.encrypt(details.get(detailName))); - } - } - } vmInstance.setDetails(details); _vmDao.saveDetails(vmInstance); } @@ -4007,11 +3998,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir } vm.setDetail(VmDetailConstants.DEPLOY_VM, "true"); - copyDiskDetailsToVm(vm, template); - - setPropertiesOnVM(vm, userVmOVFPropertiesMap); - - copyNetworkRequirementsToVm(vm, template); + persistVMDeployAsIsProperties(vm, userVmOVFPropertiesMap); _vmDao.saveDetails(vm); if (!isImport) { @@ -4056,33 +4043,15 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir }); } - private void copyNetworkRequirementsToVm(UserVmVO vm, VirtualMachineTemplate template) { - if (template.isDeployAsIs()) { - List details = templateDetailsDao.listDetailsByTemplateIdMatchingPrefix(template.getId(), ImageStore.REQUIRED_NETWORK_PREFIX); - for (VMTemplateDetailVO detail : details) { - vm.setDetail(detail.getName(), detail.getValue()); - } - } - } - - private void copyDiskDetailsToVm(UserVmVO vm, VirtualMachineTemplate template) { - if (template.isDeployAsIs()) { - List details = templateDetailsDao.listDetailsByTemplateIdMatchingPrefix(template.getId(), ImageStore.DISK_DEFINITION_PREFIX); - for (VMTemplateDetailVO detail : details) { - vm.setDetail(detail.getName(), detail.getValue()); - } - } - } - /** * take the properties and set them on the vm. * consider should we be complete, and make sure all default values are copied as well if known? * I.E. iterate over the template details as well to copy any that are not defined yet. */ - private void setPropertiesOnVM(UserVmVO vm, Map userVmOVFPropertiesMap) { + private void persistVMDeployAsIsProperties(UserVmVO vm, Map userVmOVFPropertiesMap) { if (MapUtils.isNotEmpty(userVmOVFPropertiesMap)) { for (String key : userVmOVFPropertiesMap.keySet()) { - String detailKey = ImageStore.ACS_PROPERTY_PREFIX + key; + String detailKey = key; String value = userVmOVFPropertiesMap.get(key); // Sanitize boolean values to expected format and encrypt passwords @@ -4092,7 +4061,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir } else if (value.equalsIgnoreCase("False")) { value = "False"; } else { - OVFPropertyTO propertyTO = templateDetailsDao.findPropertyByTemplateAndKey(vm.getTemplateId(), key); + OVFPropertyTO propertyTO = templateDeployAsIsDetailsDao.findPropertyByTemplateAndKey(vm.getTemplateId(), key); if (propertyTO != null && propertyTO.isPassword()) { value = DBEncryptionUtil.encrypt(value); } @@ -4101,7 +4070,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir if (s_logger.isTraceEnabled()) { s_logger.trace(String.format("setting property '%s' as '%s' with value '%s'", key, detailKey, value)); } - vm.setDetail(detailKey, value); + UserVmDeployAsIsDetailVO detail = new UserVmDeployAsIsDetailVO(vm.getId(), detailKey, value); + userVmDeployAsIsDetailsDao.persist(detail); } } } @@ -7378,19 +7348,19 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir private LinkedHashMap getVmOvfNetworkMapping(DataCenter zone, Account owner, VirtualMachineTemplate template, Map vmNetworkMapping) throws InsufficientCapacityException, ResourceAllocationException { LinkedHashMap mapping = new LinkedHashMap<>(); if (ImageFormat.OVA.equals(template.getFormat())) { - List networkPrerequisiteTOList = - templateDetailsDao.listNetworkRequirementsByTemplateId(template.getId()); - if (CollectionUtils.isNotEmpty(networkPrerequisiteTOList)) { + List OVFNetworkTOList = + templateDeployAsIsDetailsDao.listNetworkRequirementsByTemplateId(template.getId()); + if (CollectionUtils.isNotEmpty(OVFNetworkTOList)) { Network lastMappedNetwork = null; - for (NetworkPrerequisiteTO networkPrerequisiteTO : networkPrerequisiteTOList) { - Long networkId = vmNetworkMapping.get(networkPrerequisiteTO.getInstanceID()); + for (OVFNetworkTO OVFNetworkTO : OVFNetworkTOList) { + Long networkId = vmNetworkMapping.get(OVFNetworkTO.getInstanceID()); if (networkId == null && lastMappedNetwork == null) { lastMappedNetwork = getNetworkForOvfNetworkMapping(zone, owner); } if (networkId == null) { networkId = lastMappedNetwork.getId(); } - mapping.put(networkPrerequisiteTO.getInstanceID(), networkId); + mapping.put(OVFNetworkTO.getInstanceID(), networkId); } } } diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java index fec8eab8ce2..d60d7b8fb20 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java @@ -37,10 +37,10 @@ import java.util.concurrent.Executors; import javax.naming.ConfigurationException; -import com.cloud.agent.api.storage.OVFEulaSectionTO; -import com.cloud.agent.api.storage.OVFVirtualHardwareSectionTO; +import com.cloud.agent.api.to.deployasis.OVFEulaSectionTO; +import com.cloud.agent.api.to.deployasis.OVFVirtualHardwareSectionTO; import com.cloud.agent.api.to.DatadiskTO; -import com.cloud.agent.api.storage.OVFPropertyTO; +import com.cloud.agent.api.to.deployasis.OVFPropertyTO; import com.cloud.storage.template.Processor; import com.cloud.storage.template.S3TemplateDownloader; import com.cloud.storage.template.TemplateDownloader; @@ -58,7 +58,7 @@ import com.cloud.storage.template.RawImageProcessor; import com.cloud.storage.template.TARProcessor; import com.cloud.storage.template.VhdProcessor; import com.cloud.storage.template.TemplateConstants; -import org.apache.cloudstack.api.net.NetworkPrerequisiteTO; +import com.cloud.agent.api.to.deployasis.OVFNetworkTO; import org.apache.cloudstack.storage.command.DownloadCommand; import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType; import org.apache.cloudstack.storage.command.DownloadProgressCommand; @@ -132,7 +132,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager private final long id; private final ResourceType resourceType; private List ovfProperties; - private List networks; + private List networks; private List disks; private OVFVirtualHardwareSectionTO hardwareSection; private List eulaSections; @@ -239,11 +239,11 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager this.ovfProperties = ovfProperties; } - public List getNetworks() { + public List getNetworks() { return networks; } - public void setNetworks(List networks) { + public void setNetworks(List networks) { this.networks = networks; } From f73830acbbd5760dbea2acb43da357b2e8e6e2ab Mon Sep 17 00:00:00 2001 From: nvazquez Date: Mon, 14 Sep 2020 12:12:35 -0300 Subject: [PATCH 086/164] Refactor deploy as is constants --- .../java/com/cloud/deployasis/DeployAsIsConstants.java | 10 +++++----- .../dao/TemplateDeployAsIsDetailsDaoImpl.java | 4 ++-- .../storage/image/deployasis/DeployAsIsHelperImpl.java | 10 +++++----- .../com/cloud/api/query/dao/TemplateJoinDaoImpl.java | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/api/src/main/java/com/cloud/deployasis/DeployAsIsConstants.java b/api/src/main/java/com/cloud/deployasis/DeployAsIsConstants.java index 8c50a3ff401..3dc961b9de8 100644 --- a/api/src/main/java/com/cloud/deployasis/DeployAsIsConstants.java +++ b/api/src/main/java/com/cloud/deployasis/DeployAsIsConstants.java @@ -18,10 +18,10 @@ package com.cloud.deployasis; public interface DeployAsIsConstants { - String ACS_PROPERTY_PREFIX = "ACS-property-"; - String REQUIRED_NETWORK_PREFIX = "ACS-network-"; - String OVF_HARDWARE_CONFIGURATION_PREFIX = "ACS-configuration-"; - String OVF_HARDWARE_ITEM_PREFIX = "ACS-hardware-item-"; - String OVF_EULA_SECTION_PREFIX = "ACS-eula-"; + String PROPERTY_PREFIX = "property-"; + String NETWORK_PREFIX = "network-"; + String CONFIGURATION_PREFIX = "configuration-"; + String HARDWARE_ITEM_PREFIX = "hardware-item-"; + String EULA_PREFIX = "eula-"; } diff --git a/engine/schema/src/main/java/com/cloud/deployasis/dao/TemplateDeployAsIsDetailsDaoImpl.java b/engine/schema/src/main/java/com/cloud/deployasis/dao/TemplateDeployAsIsDetailsDaoImpl.java index df21d4b4872..1ffcb13b01d 100644 --- a/engine/schema/src/main/java/com/cloud/deployasis/dao/TemplateDeployAsIsDetailsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/deployasis/dao/TemplateDeployAsIsDetailsDaoImpl.java @@ -46,7 +46,7 @@ public class TemplateDeployAsIsDetailsDaoImpl extends ResourceDetailsDaoBase sc = createSearchCriteria(); sc.addAnd("templateId", SearchCriteria.Op.EQ, templateId); - sc.addAnd("name", SearchCriteria.Op.EQ, key.startsWith(DeployAsIsConstants.ACS_PROPERTY_PREFIX) ? key : DeployAsIsConstants.ACS_PROPERTY_PREFIX + key); + sc.addAnd("name", SearchCriteria.Op.EQ, key.startsWith(DeployAsIsConstants.PROPERTY_PREFIX) ? key : DeployAsIsConstants.PROPERTY_PREFIX + key); OVFPropertyTO property = null; TemplateDeployAsIsDetailVO detail = findOneBy(sc); if (detail != null) { @@ -66,7 +66,7 @@ public class TemplateDeployAsIsDetailsDaoImpl extends ResourceDetailsDaoBase listNetworkRequirementsByTemplateId(long templateId) { - List networkDetails = listDetailsByTemplateIdMatchingPrefix(templateId, DeployAsIsConstants.REQUIRED_NETWORK_PREFIX); + List networkDetails = listDetailsByTemplateIdMatchingPrefix(templateId, DeployAsIsConstants.NETWORK_PREFIX); List networkPrereqs = new ArrayList<>(); for (TemplateDeployAsIsDetailVO property : networkDetails) { OVFNetworkTO ovfPropertyTO = gson.fromJson(property.getValue(), OVFNetworkTO.class); diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/deployasis/DeployAsIsHelperImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/deployasis/DeployAsIsHelperImpl.java index 69982cd798d..fa3f0cbebc4 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/deployasis/DeployAsIsHelperImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/deployasis/DeployAsIsHelperImpl.java @@ -176,18 +176,18 @@ public class DeployAsIsHelperImpl implements DeployAsIsHelper { private String getKeyFromInformationTO(TemplateDeployAsIsInformationTO informationTO) { if (informationTO instanceof OVFPropertyTO) { - return DeployAsIsConstants.ACS_PROPERTY_PREFIX + ((OVFPropertyTO) informationTO).getKey(); + return DeployAsIsConstants.PROPERTY_PREFIX + ((OVFPropertyTO) informationTO).getKey(); } else if (informationTO instanceof OVFNetworkTO) { - return DeployAsIsConstants.REQUIRED_NETWORK_PREFIX + ((OVFNetworkTO) informationTO).getName(); + return DeployAsIsConstants.NETWORK_PREFIX + ((OVFNetworkTO) informationTO).getName(); } else if (informationTO instanceof OVFConfigurationTO) { - return DeployAsIsConstants.OVF_HARDWARE_CONFIGURATION_PREFIX + + return DeployAsIsConstants.CONFIGURATION_PREFIX + ((OVFConfigurationTO) informationTO).getIndex() + "-" + ((OVFConfigurationTO) informationTO).getId(); } else if (informationTO instanceof OVFVirtualHardwareItemTO) { String key = ((OVFVirtualHardwareItemTO) informationTO).getResourceType().getName().trim().replaceAll("\\s","") + "-" + ((OVFVirtualHardwareItemTO) informationTO).getInstanceId(); - return DeployAsIsConstants.OVF_HARDWARE_ITEM_PREFIX + key; + return DeployAsIsConstants.HARDWARE_ITEM_PREFIX + key; } else if (informationTO instanceof OVFEulaSectionTO) { - return DeployAsIsConstants.OVF_EULA_SECTION_PREFIX + ((OVFEulaSectionTO) informationTO).getIndex() + + return DeployAsIsConstants.EULA_PREFIX + ((OVFEulaSectionTO) informationTO).getIndex() + "-" + ((OVFEulaSectionTO) informationTO).getInfo(); } return null; diff --git a/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDaoImpl.java index 9cfe710fd71..b96b42d1339 100644 --- a/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDaoImpl.java @@ -283,7 +283,7 @@ public class TemplateJoinDaoImpl extends GenericDaoBaseWithTagInformation deployAsIsDetails = templateDeployAsIsDetailsDao.listDetails(template.getId()); for (TemplateDeployAsIsDetailVO deployAsIsDetailVO : deployAsIsDetails) { - if (deployAsIsDetailVO.getName().startsWith(DeployAsIsConstants.OVF_HARDWARE_ITEM_PREFIX)) { + if (deployAsIsDetailVO.getName().startsWith(DeployAsIsConstants.HARDWARE_ITEM_PREFIX)) { //Do not list hardware items continue; } From 44bae9866428539b388578487364f3893f0951c1 Mon Sep 17 00:00:00 2001 From: nvazquez Date: Mon, 14 Sep 2020 14:57:01 -0300 Subject: [PATCH 087/164] Fix daos issue --- .../TemplateDeployAsIsDetailVO.java | 10 +- ...ateDeployAsIsDetailVO.java~Stashed changes | 94 ------------------- .../deployasis/UserVmDeployAsIsDetailVO.java | 10 +- ...rVmDeployAsIsDetailVO.java~Stashed changes | 94 ------------------- .../dao/TemplateDeployAsIsDetailsDaoImpl.java | 4 +- ...loyAsIsDetailsDaoImpl.java~Stashed changes | 83 ---------------- 6 files changed, 12 insertions(+), 283 deletions(-) delete mode 100644 engine/schema/src/main/java/com/cloud/deployasis/TemplateDeployAsIsDetailVO.java~Stashed changes delete mode 100644 engine/schema/src/main/java/com/cloud/deployasis/UserVmDeployAsIsDetailVO.java~Stashed changes delete mode 100644 engine/schema/src/main/java/com/cloud/deployasis/dao/TemplateDeployAsIsDetailsDaoImpl.java~Stashed changes diff --git a/engine/schema/src/main/java/com/cloud/deployasis/TemplateDeployAsIsDetailVO.java b/engine/schema/src/main/java/com/cloud/deployasis/TemplateDeployAsIsDetailVO.java index 00a69cd0b0c..047d985d0bb 100644 --- a/engine/schema/src/main/java/com/cloud/deployasis/TemplateDeployAsIsDetailVO.java +++ b/engine/schema/src/main/java/com/cloud/deployasis/TemplateDeployAsIsDetailVO.java @@ -36,7 +36,7 @@ public class TemplateDeployAsIsDetailVO implements ResourceDetail { private long id; @Column(name = "template_id") - private long templateId; + private long resourceId; @Column(name = "name") private String name; @@ -49,7 +49,7 @@ public class TemplateDeployAsIsDetailVO implements ResourceDetail { } public TemplateDeployAsIsDetailVO(long templateId, String name, String value) { - this.templateId = templateId; + this.resourceId = templateId; this.name = name; this.value = value; } @@ -60,7 +60,7 @@ public class TemplateDeployAsIsDetailVO implements ResourceDetail { } public long getResourceId() { - return templateId; + return resourceId; } public String getName() { @@ -80,8 +80,8 @@ public class TemplateDeployAsIsDetailVO implements ResourceDetail { this.id = id; } - public void setTemplateId(long resourceId) { - this.templateId = resourceId; + public void setResourceId(long resourceId) { + this.resourceId = resourceId; } public void setName(String name) { diff --git a/engine/schema/src/main/java/com/cloud/deployasis/TemplateDeployAsIsDetailVO.java~Stashed changes b/engine/schema/src/main/java/com/cloud/deployasis/TemplateDeployAsIsDetailVO.java~Stashed changes deleted file mode 100644 index 047d985d0bb..00000000000 --- a/engine/schema/src/main/java/com/cloud/deployasis/TemplateDeployAsIsDetailVO.java~Stashed changes +++ /dev/null @@ -1,94 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -package com.cloud.deployasis; - -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.GeneratedValue; -import javax.persistence.GenerationType; -import javax.persistence.Id; -import javax.persistence.Lob; -import javax.persistence.Table; - -import org.apache.cloudstack.api.ResourceDetail; - -@Entity -@Table(name = "template_deploy_as_is_details") -public class TemplateDeployAsIsDetailVO implements ResourceDetail { - - @Id - @GeneratedValue(strategy = GenerationType.IDENTITY) - @Column(name = "id") - private long id; - - @Column(name = "template_id") - private long resourceId; - - @Column(name = "name") - private String name; - - @Lob - @Column(name = "value", length = 65535) - private String value; - - public TemplateDeployAsIsDetailVO() { - } - - public TemplateDeployAsIsDetailVO(long templateId, String name, String value) { - this.resourceId = templateId; - this.name = name; - this.value = value; - } - - @Override - public long getId() { - return id; - } - - public long getResourceId() { - return resourceId; - } - - public String getName() { - return name; - } - - public String getValue() { - return value; - } - - @Override - public boolean isDisplay() { - return true; - } - - public void setId(long id) { - this.id = id; - } - - public void setResourceId(long resourceId) { - this.resourceId = resourceId; - } - - public void setName(String name) { - this.name = name; - } - - public void setValue(String value) { - this.value = value; - } -} diff --git a/engine/schema/src/main/java/com/cloud/deployasis/UserVmDeployAsIsDetailVO.java b/engine/schema/src/main/java/com/cloud/deployasis/UserVmDeployAsIsDetailVO.java index 6cbc22b6b38..b56b4a49464 100644 --- a/engine/schema/src/main/java/com/cloud/deployasis/UserVmDeployAsIsDetailVO.java +++ b/engine/schema/src/main/java/com/cloud/deployasis/UserVmDeployAsIsDetailVO.java @@ -36,7 +36,7 @@ public class UserVmDeployAsIsDetailVO implements ResourceDetail { private long id; @Column(name = "vm_id") - private long vmId; + private long resourceId; @Column(name = "name") private String name; @@ -49,7 +49,7 @@ public class UserVmDeployAsIsDetailVO implements ResourceDetail { } public UserVmDeployAsIsDetailVO(long vmId, String name, String value) { - this.vmId = vmId; + this.resourceId = vmId; this.name = name; this.value = value; } @@ -60,7 +60,7 @@ public class UserVmDeployAsIsDetailVO implements ResourceDetail { } public long getResourceId() { - return vmId; + return resourceId; } public String getName() { @@ -80,8 +80,8 @@ public class UserVmDeployAsIsDetailVO implements ResourceDetail { this.id = id; } - public void setVmId(long resourceId) { - this.vmId = resourceId; + public void setResourceId(long resourceId) { + this.resourceId = resourceId; } public void setName(String name) { diff --git a/engine/schema/src/main/java/com/cloud/deployasis/UserVmDeployAsIsDetailVO.java~Stashed changes b/engine/schema/src/main/java/com/cloud/deployasis/UserVmDeployAsIsDetailVO.java~Stashed changes deleted file mode 100644 index b56b4a49464..00000000000 --- a/engine/schema/src/main/java/com/cloud/deployasis/UserVmDeployAsIsDetailVO.java~Stashed changes +++ /dev/null @@ -1,94 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -package com.cloud.deployasis; - -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.GeneratedValue; -import javax.persistence.GenerationType; -import javax.persistence.Id; -import javax.persistence.Lob; -import javax.persistence.Table; - -import org.apache.cloudstack.api.ResourceDetail; - -@Entity -@Table(name = "user_vm_deploy_as_is_details") -public class UserVmDeployAsIsDetailVO implements ResourceDetail { - - @Id - @GeneratedValue(strategy = GenerationType.IDENTITY) - @Column(name = "id") - private long id; - - @Column(name = "vm_id") - private long resourceId; - - @Column(name = "name") - private String name; - - @Lob - @Column(name = "value", length = 65535) - private String value; - - public UserVmDeployAsIsDetailVO() { - } - - public UserVmDeployAsIsDetailVO(long vmId, String name, String value) { - this.resourceId = vmId; - this.name = name; - this.value = value; - } - - @Override - public long getId() { - return id; - } - - public long getResourceId() { - return resourceId; - } - - public String getName() { - return name; - } - - public String getValue() { - return value; - } - - @Override - public boolean isDisplay() { - return true; - } - - public void setId(long id) { - this.id = id; - } - - public void setResourceId(long resourceId) { - this.resourceId = resourceId; - } - - public void setName(String name) { - this.name = name; - } - - public void setValue(String value) { - this.value = value; - } -} diff --git a/engine/schema/src/main/java/com/cloud/deployasis/dao/TemplateDeployAsIsDetailsDaoImpl.java b/engine/schema/src/main/java/com/cloud/deployasis/dao/TemplateDeployAsIsDetailsDaoImpl.java index 1ffcb13b01d..3485afa34cf 100644 --- a/engine/schema/src/main/java/com/cloud/deployasis/dao/TemplateDeployAsIsDetailsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/deployasis/dao/TemplateDeployAsIsDetailsDaoImpl.java @@ -45,7 +45,7 @@ public class TemplateDeployAsIsDetailsDaoImpl extends ResourceDetailsDaoBase sc = createSearchCriteria(); - sc.addAnd("templateId", SearchCriteria.Op.EQ, templateId); + sc.addAnd("resourceId", SearchCriteria.Op.EQ, templateId); sc.addAnd("name", SearchCriteria.Op.EQ, key.startsWith(DeployAsIsConstants.PROPERTY_PREFIX) ? key : DeployAsIsConstants.PROPERTY_PREFIX + key); OVFPropertyTO property = null; TemplateDeployAsIsDetailVO detail = findOneBy(sc); @@ -58,7 +58,7 @@ public class TemplateDeployAsIsDetailsDaoImpl extends ResourceDetailsDaoBase listDetailsByTemplateIdMatchingPrefix(long templateId, String prefix) { SearchCriteria ssc = createSearchCriteria(); - ssc.addAnd("templateId", SearchCriteria.Op.EQ, templateId); + ssc.addAnd("resouceId", SearchCriteria.Op.EQ, templateId); ssc.addAnd("name", SearchCriteria.Op.LIKE, prefix + "%"); return search(ssc, null); diff --git a/engine/schema/src/main/java/com/cloud/deployasis/dao/TemplateDeployAsIsDetailsDaoImpl.java~Stashed changes b/engine/schema/src/main/java/com/cloud/deployasis/dao/TemplateDeployAsIsDetailsDaoImpl.java~Stashed changes deleted file mode 100644 index 6f4ec01d1f4..00000000000 --- a/engine/schema/src/main/java/com/cloud/deployasis/dao/TemplateDeployAsIsDetailsDaoImpl.java~Stashed changes +++ /dev/null @@ -1,83 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -package com.cloud.deployasis.dao; - -import com.cloud.agent.api.to.deployasis.OVFPropertyTO; -import com.cloud.deployasis.DeployAsIsConstants; -import com.cloud.deployasis.TemplateDeployAsIsDetailVO; -import com.cloud.utils.db.SearchCriteria; -import com.google.gson.Gson; -import com.cloud.agent.api.to.deployasis.OVFNetworkTO; -import org.apache.cloudstack.resourcedetail.ResourceDetailsDaoBase; -import org.springframework.stereotype.Component; - -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; - -@Component -public class TemplateDeployAsIsDetailsDaoImpl extends ResourceDetailsDaoBase implements TemplateDeployAsIsDetailsDao { - - private Gson gson = new Gson(); - - public TemplateDeployAsIsDetailsDaoImpl() { - } - - @Override - public void addDetail(long resourceId, String key, String value, boolean display) { - super.addDetail(new TemplateDeployAsIsDetailVO(resourceId, key, value)); - } - - @Override - public OVFPropertyTO findPropertyByTemplateAndKey(long templateId, String key) { - SearchCriteria sc = createSearchCriteria(); - sc.addAnd("resourceId", SearchCriteria.Op.EQ, templateId); - sc.addAnd("name", SearchCriteria.Op.EQ, key.startsWith(DeployAsIsConstants.ACS_PROPERTY_PREFIX) ? key : DeployAsIsConstants.ACS_PROPERTY_PREFIX + key); - OVFPropertyTO property = null; - TemplateDeployAsIsDetailVO detail = findOneBy(sc); - if (detail != null) { - property = gson.fromJson(detail.getValue(), OVFPropertyTO.class); - } - return property; - } - - @Override - public List listDetailsByTemplateIdMatchingPrefix(long templateId, String prefix) { - SearchCriteria ssc = createSearchCriteria(); - ssc.addAnd("resouceId", SearchCriteria.Op.EQ, templateId); - ssc.addAnd("name", SearchCriteria.Op.LIKE, prefix + "%"); - - return search(ssc, null); - } - - @Override - public List listNetworkRequirementsByTemplateId(long templateId) { - List networkDetails = listDetailsByTemplateIdMatchingPrefix(templateId, DeployAsIsConstants.REQUIRED_NETWORK_PREFIX); - List networkPrereqs = new ArrayList<>(); - for (TemplateDeployAsIsDetailVO property : networkDetails) { - OVFNetworkTO ovfPropertyTO = gson.fromJson(property.getValue(), OVFNetworkTO.class); - networkPrereqs.add(ovfPropertyTO); - } - networkPrereqs.sort(new Comparator() { - @Override - public int compare(OVFNetworkTO o1, OVFNetworkTO o2) { - return o1.getInstanceID() - o2.getInstanceID(); - } - }); - return networkPrereqs; - } -} From 818981d8a7ef703125bc56d81c36b92653ba80a8 Mon Sep 17 00:00:00 2001 From: nvazquez Date: Mon, 14 Sep 2020 15:26:12 -0300 Subject: [PATCH 088/164] Fix typo --- .../cloud/deployasis/dao/TemplateDeployAsIsDetailsDaoImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/schema/src/main/java/com/cloud/deployasis/dao/TemplateDeployAsIsDetailsDaoImpl.java b/engine/schema/src/main/java/com/cloud/deployasis/dao/TemplateDeployAsIsDetailsDaoImpl.java index 3485afa34cf..198ef6db89b 100644 --- a/engine/schema/src/main/java/com/cloud/deployasis/dao/TemplateDeployAsIsDetailsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/deployasis/dao/TemplateDeployAsIsDetailsDaoImpl.java @@ -58,7 +58,7 @@ public class TemplateDeployAsIsDetailsDaoImpl extends ResourceDetailsDaoBase listDetailsByTemplateIdMatchingPrefix(long templateId, String prefix) { SearchCriteria ssc = createSearchCriteria(); - ssc.addAnd("resouceId", SearchCriteria.Op.EQ, templateId); + ssc.addAnd("resourceId", SearchCriteria.Op.EQ, templateId); ssc.addAnd("name", SearchCriteria.Op.LIKE, prefix + "%"); return search(ssc, null); From 6ebfdc68545f66539010905b59973a3ed18c7b65 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Wed, 16 Sep 2020 10:50:56 +0530 Subject: [PATCH 089/164] Dettached datadisk migration failed between NFS storages. Fixed the issue of worker VM name being null --- .../com/cloud/hypervisor/vmware/resource/VmwareResource.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index f4d2c6d424c..1bf2c66b80b 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -4752,8 +4752,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa dsMo = new DatastoreMO(hyperHost.getContext(), morSourceDS); morDestintionDS = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, cmd.getTargetPool().getUuid()); destinationDsMo = new DatastoreMO(hyperHost.getContext(), morDestintionDS); - if (dsMo.getDatastoreType().equalsIgnoreCase("VVOL")) - vmName = getWorkerName(getServiceContext(), cmd, 0, dsMo); + vmName = getWorkerName(getServiceContext(), cmd, 0, dsMo); if (destinationDsMo.getDatastoreType().equalsIgnoreCase("VVOL")) vmName = getWorkerName(getServiceContext(), cmd, 0, destinationDsMo); From 19c83fef641954d0f8d4bd982444fc8d222fe9c9 Mon Sep 17 00:00:00 2001 From: nvazquez Date: Wed, 16 Sep 2020 10:36:00 -0300 Subject: [PATCH 090/164] Fix zone to zone scope volume migrations NPE --- .../storage/motion/VmwareStorageMotionStrategy.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/storage/motion/VmwareStorageMotionStrategy.java b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/storage/motion/VmwareStorageMotionStrategy.java index 2463e75c01d..5a7b4c4ca67 100644 --- a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/storage/motion/VmwareStorageMotionStrategy.java +++ b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/storage/motion/VmwareStorageMotionStrategy.java @@ -132,7 +132,10 @@ public class VmwareStorageMotionStrategy implements DataMotionStrategy { StoragePool srcPool = storagePoolDao.findById(srcStore.getId()); DataStore destStore = destData.getDataStore(); StoragePool destPool = storagePoolDao.findById(destStore.getId()); - return srcPool.getClusterId().equals(destPool.getClusterId()); + if (srcPool.getClusterId() != null && destPool.getClusterId() != null) { + return srcPool.getClusterId().equals(destPool.getClusterId()); + } + return false; } /** From 44dc0c607288cee1e2db4194ac106c3399ded064 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Thu, 17 Sep 2020 11:54:37 +0530 Subject: [PATCH 091/164] Fixed rat failure on new class DeployAsIsHelper.java Also removed some unused imports during rebase --- .../image/deployasis/DeployAsIsHelper.java | 16 +++++++++ .../vmware/resource/VmwareResource.java | 36 ------------------- 2 files changed, 16 insertions(+), 36 deletions(-) diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/deployasis/DeployAsIsHelper.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/deployasis/DeployAsIsHelper.java index b23326b0406..574e8cdf7bd 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/deployasis/DeployAsIsHelper.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/deployasis/DeployAsIsHelper.java @@ -1,3 +1,19 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. package org.apache.cloudstack.storage.image.deployasis; import com.cloud.agent.api.storage.DownloadAnswer; diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 1bf2c66b80b..841a1779d59 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -345,42 +345,6 @@ import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.command.StorageSubSystemCommand; import org.apache.cloudstack.storage.resource.NfsSecondaryStorageResource; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; -import org.apache.cloudstack.storage.to.VolumeObjectTO; -import org.apache.cloudstack.utils.volume.VirtualMachineDiskInfo; -import org.apache.cloudstack.vm.UnmanagedInstanceTO; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang.math.NumberUtils; -import org.apache.log4j.Logger; -import org.apache.log4j.NDC; -import org.joda.time.Duration; - -import javax.naming.ConfigurationException; -import javax.xml.datatype.XMLGregorianCalendar; -import java.io.File; -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.net.ConnectException; -import java.net.InetSocketAddress; -import java.net.URI; -import java.net.URL; -import java.nio.channels.SocketChannel; -import java.rmi.RemoteException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.Date; -import java.util.EnumMap; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.Set; -import java.util.TimeZone; -import java.util.UUID; import static com.cloud.utils.HumanReadableJson.getHumanReadableBytesJson; import static com.cloud.utils.NumbersUtil.toHumanReadableSize; From 33ae2afc8979b0d9f6b749c5ab8ceb731050f7b1 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Thu, 17 Sep 2020 13:49:31 +0530 Subject: [PATCH 092/164] Removed few duplicate imports during rebase with master --- .../org/apache/cloudstack/storage/volume/VolumeObject.java | 1 - .../main/java/com/cloud/api/query/dao/TemplateJoinDaoImpl.java | 3 --- 2 files changed, 4 deletions(-) diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java index d2a771d46a5..45509c51c43 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java @@ -47,7 +47,6 @@ import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.offering.DiskOffering.DiskCacheMode; import com.cloud.storage.DataStoreRole; import com.cloud.storage.DiskOfferingVO; -import com.cloud.storage.MigrationOptions; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.Storage.ProvisioningType; import com.cloud.storage.Volume; diff --git a/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDaoImpl.java index b96b42d1339..92072c7a3c5 100644 --- a/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDaoImpl.java @@ -44,9 +44,6 @@ import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; -import org.apache.cloudstack.utils.security.DigestHelper; -import org.apache.log4j.Logger; -import org.springframework.stereotype.Component; import com.cloud.api.ApiDBUtils; import com.cloud.api.ApiResponseHelper; From 745dd00c9aaaeeb9dc30eeb3959e29c997aea204 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Thu, 17 Sep 2020 16:13:00 +0530 Subject: [PATCH 093/164] Remove duplicate schema changes --- .../META-INF/db/schema-41400to41500.sql | 68 ------------------- 1 file changed, 68 deletions(-) diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql b/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql index 12c2a8e12ba..a868e2a21ea 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql @@ -473,74 +473,6 @@ ADD CONSTRAINT `fk_template_spool_ref__template_id` ON DELETE NO ACTION ON UPDATE NO ACTION; - -CREATE TABLE IF NOT EXISTS `cloud`.`vsphere_storage_policy` ( - `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT, - `uuid` varchar(255) UNIQUE, - `zone_id` bigint(20) unsigned NOT NULL COMMENT 'id of the zone', - `policy_id` varchar(255) NOT NULL COMMENT 'the identifier of the Storage Policy in vSphere DataCenter', - `name` varchar(255) NOT NULL COMMENT 'name of the storage policy', - `description` text COMMENT 'description of the storage policy', - `update_time` datetime COMMENT 'last updated when policy imported', - `removed` datetime COMMENT 'date removed', - PRIMARY KEY (`id`), - KEY `fk_vsphere_storage_policy__zone_id` (`zone_id`), - UNIQUE KEY (`zone_id`, `policy_id`), - CONSTRAINT `fk_vsphere_storage_policy__zone_id` FOREIGN KEY (`zone_id`) REFERENCES `data_center` (`id`) ON DELETE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=utf8; - -ALTER TABLE `cloud`.`storage_pool` ADD COLUMN `parent` BIGINT(20) UNSIGNED NOT NULL DEFAULT 0 COMMENT 'ID of the Datastore cluster (storage pool) if this is a child in that Datastore cluster'; - --- Added parent column to support datastore clusters in vmware vsphere -DROP VIEW IF EXISTS `cloud`.`storage_pool_view`; -CREATE VIEW `cloud`.`storage_pool_view` AS - SELECT - `storage_pool`.`id` AS `id`, - `storage_pool`.`uuid` AS `uuid`, - `storage_pool`.`name` AS `name`, - `storage_pool`.`status` AS `status`, - `storage_pool`.`path` AS `path`, - `storage_pool`.`pool_type` AS `pool_type`, - `storage_pool`.`host_address` AS `host_address`, - `storage_pool`.`created` AS `created`, - `storage_pool`.`removed` AS `removed`, - `storage_pool`.`capacity_bytes` AS `capacity_bytes`, - `storage_pool`.`capacity_iops` AS `capacity_iops`, - `storage_pool`.`scope` AS `scope`, - `storage_pool`.`hypervisor` AS `hypervisor`, - `storage_pool`.`storage_provider_name` AS `storage_provider_name`, - `storage_pool`.`parent` AS `parent`, - `cluster`.`id` AS `cluster_id`, - `cluster`.`uuid` AS `cluster_uuid`, - `cluster`.`name` AS `cluster_name`, - `cluster`.`cluster_type` AS `cluster_type`, - `data_center`.`id` AS `data_center_id`, - `data_center`.`uuid` AS `data_center_uuid`, - `data_center`.`name` AS `data_center_name`, - `data_center`.`networktype` AS `data_center_type`, - `host_pod_ref`.`id` AS `pod_id`, - `host_pod_ref`.`uuid` AS `pod_uuid`, - `host_pod_ref`.`name` AS `pod_name`, - `storage_pool_tags`.`tag` AS `tag`, - `op_host_capacity`.`used_capacity` AS `disk_used_capacity`, - `op_host_capacity`.`reserved_capacity` AS `disk_reserved_capacity`, - `async_job`.`id` AS `job_id`, - `async_job`.`uuid` AS `job_uuid`, - `async_job`.`job_status` AS `job_status`, - `async_job`.`account_id` AS `job_account_id` - FROM - ((((((`storage_pool` - LEFT JOIN `cluster` ON ((`storage_pool`.`cluster_id` = `cluster`.`id`))) - LEFT JOIN `data_center` ON ((`storage_pool`.`data_center_id` = `data_center`.`id`))) - LEFT JOIN `host_pod_ref` ON ((`storage_pool`.`pod_id` = `host_pod_ref`.`id`))) - LEFT JOIN `storage_pool_tags` ON (((`storage_pool_tags`.`pool_id` = `storage_pool`.`id`)))) - LEFT JOIN `op_host_capacity` ON (((`storage_pool`.`id` = `op_host_capacity`.`host_id`) - AND (`op_host_capacity`.`capacity_type` IN (3 , 9))))) - LEFT JOIN `async_job` ON (((`async_job`.`instance_id` = `storage_pool`.`id`) - AND (`async_job`.`instance_type` = 'StoragePool') - AND (`async_job`.`job_status` = 0)))); - - CREATE TABLE `cloud`.`template_deploy_as_is_details` ( `id` bigint unsigned NOT NULL auto_increment, `template_id` bigint unsigned NOT NULL COMMENT 'template id', From 9c162c6de9d79cdd6f4aa50f95dfa0298bc57a65 Mon Sep 17 00:00:00 2001 From: nvazquez Date: Thu, 17 Sep 2020 16:07:55 -0300 Subject: [PATCH 094/164] Fix ISO upload on Deploy OVF task --- .../java/com/cloud/hypervisor/vmware/util/VmwareContext.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareContext.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareContext.java index ec11899ad3b..af44962322e 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareContext.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareContext.java @@ -427,6 +427,9 @@ public class VmwareContext { conn.setChunkedStreamingMode(ChunkSize); conn.setRequestMethod(httpMethod); + if (urlString.endsWith(".iso")) { + conn.setRequestProperty("Overwrite", "t"); + } conn.setRequestProperty("Connection", "Keep-Alive"); String contentType = urlString.endsWith(".iso") ? "application/octet-stream" : From 61e7625d49e484ba6af855ca7b19c9f0fc5521ac Mon Sep 17 00:00:00 2001 From: nvazquez Date: Sat, 19 Sep 2020 23:23:59 -0300 Subject: [PATCH 095/164] Do not set user-configurable details for deploy-as-is VMs --- .../vmware/resource/VmwareResource.java | 680 ++++++++++-------- .../vmware/mo/HypervisorHostHelper.java | 2 +- .../hypervisor/vmware/util/VmwareHelper.java | 8 +- 3 files changed, 370 insertions(+), 320 deletions(-) diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 841a1779d59..e2db99fea4b 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -1749,38 +1749,15 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa VirtualMachineFileLayoutEx existingVmFileLayout = null; List existingDatastores = new ArrayList(); + DeployAsIsInfoTO deployAsIsInfo = vmSpec.getDeployAsIsInfo(); + boolean deployAsIs = deployAsIsInfo != null; + Pair names = composeVmNames(vmSpec); String vmInternalCSName = names.first(); String vmNameOnVcenter = names.second(); - String dataDiskController = vmSpec.getDetails().get(VmDetailConstants.DATA_DISK_CONTROLLER); - String rootDiskController = vmSpec.getDetails().get(VmDetailConstants.ROOT_DISK_CONTROLLER); DiskTO rootDiskTO = null; - String bootMode = null; - if (vmSpec.getDetails().containsKey(VmDetailConstants.BOOT_MODE)) { - bootMode = vmSpec.getDetails().get(VmDetailConstants.BOOT_MODE); - } - if (null == bootMode) { - bootMode = ApiConstants.BootType.BIOS.toString(); - } - - // If root disk controller is scsi, then data disk controller would also be scsi instead of using 'osdefault' - // This helps avoid mix of different scsi subtype controllers in instance. - if (DiskControllerType.osdefault == DiskControllerType.getType(dataDiskController) && DiskControllerType.lsilogic == DiskControllerType.getType(rootDiskController)) { - dataDiskController = DiskControllerType.scsi.toString(); - } - - // Validate the controller types - dataDiskController = DiskControllerType.getType(dataDiskController).toString(); - rootDiskController = DiskControllerType.getType(rootDiskController).toString(); - - if (DiskControllerType.getType(rootDiskController) == DiskControllerType.none) { - throw new CloudRuntimeException("Invalid root disk controller detected : " + rootDiskController); - } - if (DiskControllerType.getType(dataDiskController) == DiskControllerType.none) { - throw new CloudRuntimeException("Invalid data disk controller detected : " + dataDiskController); - } - - Pair controllerInfo = new Pair(rootDiskController, dataDiskController); + String bootMode = getBootModeFromVmSpec(vmSpec, deployAsIs); + Pair controllerInfo = getControllerInfoFromVmSpec(vmSpec, deployAsIs); Boolean systemVm = vmSpec.getType().isUsedBySystem(); // Thus, vmInternalCSName always holds i-x-y, the cloudstack generated internal VM name. @@ -1802,19 +1779,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } DiskTO[] specDisks = vmSpec.getDisks(); - DeployAsIsInfoTO deployAsIsInfo = vmSpec.getDeployAsIsInfo(); - boolean installAsIs = deployAsIsInfo != null; - if (installAsIs && dcMo.findVm(vmInternalCSName) == null) { - if (s_logger.isTraceEnabled()) { - s_logger.trace("Deploying OVA from as is"); - } - String deployAsIsTemplate = deployAsIsInfo.getTemplatePath(); - String destDatastore = deployAsIsInfo.getDestStoragePool(); - vmInVcenter = _storageProcessor.cloneVMFromTemplate(deployAsIsTemplate, vmInternalCSName, destDatastore); - mapSpecDisksToClonedDisks(vmInVcenter, vmInternalCSName, specDisks); - } - - String guestOsId = translateGuestOsIdentifier(vmSpec.getArch(), vmSpec.getOs(), vmSpec.getPlatformEmulator()).value(); + String guestOsId = getGuestOsIdFromVmSpec(vmSpec, deployAsIs); DiskTO[] disks = validateDisks(vmSpec.getDisks()); assert (disks.length > 0); NicTO[] nics = vmSpec.getNics(); @@ -1840,6 +1805,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa int firstScsiControllerBusNum = 0; int numScsiControllerForSystemVm = 1; boolean hasSnapshot = false; + if (vmMo != null) { s_logger.info("VM " + vmInternalCSName + " already exists, tear down devices for reconfiguration"); if (getVmPowerState(vmMo) != PowerState.PowerOff) @@ -1849,15 +1815,10 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa diskInfoBuilder = vmMo.getDiskInfoBuilder(); hasSnapshot = vmMo.hasSnapshot(); nicDevices = vmMo.getNicDevices(); - if (!hasSnapshot) - vmMo.tearDownDevices(new Class[]{VirtualDisk.class, VirtualEthernetCard.class}); - else - vmMo.tearDownDevices(new Class[]{VirtualEthernetCard.class}); - if (systemVm) { - ensureScsiDiskControllers(vmMo, systemVmScsiControllerType.toString(), numScsiControllerForSystemVm, firstScsiControllerBusNum); - } else { - ensureDiskControllers(vmMo, controllerInfo); - } + + tearDownVmDevices(vmMo, hasSnapshot, deployAsIs); + ensureDiskControllersInternal(vmMo, systemVm, controllerInfo, systemVmScsiControllerType, + numScsiControllerForSystemVm, firstScsiControllerBusNum, deployAsIs); } else { ManagedObjectReference morDc = hyperHost.getHyperHostDatacenter(); assert (morDc != null); @@ -1875,17 +1836,10 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa diskInfoBuilder = vmMo.getDiskInfoBuilder(); hasSnapshot = vmMo.hasSnapshot(); - if (!hasSnapshot) - vmMo.tearDownDevices(new Class[]{VirtualDisk.class, VirtualEthernetCard.class}); - else - vmMo.tearDownDevices(new Class[]{VirtualEthernetCard.class}); - if (systemVm) { - // System volumes doesn't require more than 1 SCSI controller as there is no requirement for data volumes. - ensureScsiDiskControllers(vmMo, systemVmScsiControllerType.toString(), numScsiControllerForSystemVm, firstScsiControllerBusNum); - } else { - ensureDiskControllers(vmMo, controllerInfo); - } + tearDownVmDevices(vmMo, hasSnapshot, deployAsIs); + ensureDiskControllersInternal(vmMo, systemVm, controllerInfo, systemVmScsiControllerType, + numScsiControllerForSystemVm, firstScsiControllerBusNum, deployAsIs); } else { // If a VM with the same name is found in a different cluster in the DC, unregister the old VM and configure a new VM (cold-migration). VirtualMachineMO existingVmInDc = dcMo.findVm(vmInternalCSName); @@ -1897,45 +1851,56 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa existingDatastores = existingVmInDc.getAllDatastores(); existingVmInDc.unregisterVm(); } - Pair rootDiskDataStoreDetails = null; - for (DiskTO vol : disks) { - if (vol.getType() == Volume.Type.ROOT) { - Map details = vol.getDetails(); - boolean managed = false; - if (details != null) { - managed = Boolean.parseBoolean(details.get(DiskTO.MANAGED)); - } + if (deployAsIs) { + if (s_logger.isTraceEnabled()) { + s_logger.trace("Deploying OVA from template as-is"); + } + String deployAsIsTemplate = deployAsIsInfo.getTemplatePath(); + String destDatastore = deployAsIsInfo.getDestStoragePool(); + vmMo = _storageProcessor.cloneVMFromTemplate(deployAsIsTemplate, vmInternalCSName, destDatastore); + mapSpecDisksToClonedDisks(vmMo, vmInternalCSName, specDisks); + } else { + Pair rootDiskDataStoreDetails = null; + for (DiskTO vol : disks) { + if (vol.getType() == Volume.Type.ROOT) { + Map details = vol.getDetails(); + boolean managed = false; - if (managed) { - String datastoreName = VmwareResource.getDatastoreName(details.get(DiskTO.IQN)); + if (details != null) { + managed = Boolean.parseBoolean(details.get(DiskTO.MANAGED)); + } - rootDiskDataStoreDetails = dataStoresDetails.get(datastoreName); - } else { - DataStoreTO primaryStore = vol.getData().getDataStore(); + if (managed) { + String datastoreName = VmwareResource.getDatastoreName(details.get(DiskTO.IQN)); - rootDiskDataStoreDetails = dataStoresDetails.get(primaryStore.getUuid()); + rootDiskDataStoreDetails = dataStoresDetails.get(datastoreName); + } else { + DataStoreTO primaryStore = vol.getData().getDataStore(); + + rootDiskDataStoreDetails = dataStoresDetails.get(primaryStore.getUuid()); + } } } - } - assert (vmSpec.getMinSpeed() != null) && (rootDiskDataStoreDetails != null); + assert (vmSpec.getMinSpeed() != null) && (rootDiskDataStoreDetails != null); - boolean vmFolderExists = rootDiskDataStoreDetails.second().folderExists(String.format("[%s]", rootDiskDataStoreDetails.second().getName()), vmNameOnVcenter); - String vmxFileFullPath = dsRootVolumeIsOn.searchFileInSubFolders(vmNameOnVcenter + ".vmx", false, VmwareManager.s_vmwareSearchExcludeFolder.value()); - if (vmFolderExists && vmxFileFullPath != null) { // VM can be registered only if .vmx is present. - registerVm(vmNameOnVcenter, dsRootVolumeIsOn); - vmMo = hyperHost.findVmOnHyperHost(vmInternalCSName); - if (vmMo != null) { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Found registered vm " + vmInternalCSName + " at host " + hyperHost.getHyperHostName()); + boolean vmFolderExists = rootDiskDataStoreDetails.second().folderExists(String.format("[%s]", rootDiskDataStoreDetails.second().getName()), vmNameOnVcenter); + String vmxFileFullPath = dsRootVolumeIsOn.searchFileInSubFolders(vmNameOnVcenter + ".vmx", false, VmwareManager.s_vmwareSearchExcludeFolder.value()); + if (vmFolderExists && vmxFileFullPath != null) { // VM can be registered only if .vmx is present. + registerVm(vmNameOnVcenter, dsRootVolumeIsOn); + vmMo = hyperHost.findVmOnHyperHost(vmInternalCSName); + if (vmMo != null) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Found registered vm " + vmInternalCSName + " at host " + hyperHost.getHyperHostName()); + } } + tearDownVm(vmMo); + } else if (!hyperHost.createBlankVm(vmNameOnVcenter, vmInternalCSName, vmSpec.getCpus(), vmSpec.getMaxSpeed().intValue(), getReservedCpuMHZ(vmSpec), + vmSpec.getLimitCpuUse(), (int) (vmSpec.getMaxRam() / ResourceType.bytesToMiB), getReservedMemoryMb(vmSpec), guestOsId, rootDiskDataStoreDetails.first(), false, + controllerInfo, systemVm)) { + throw new Exception("Failed to create VM. vmName: " + vmInternalCSName); } - tearDownVm(vmMo); - } else if (!hyperHost.createBlankVm(vmNameOnVcenter, vmInternalCSName, vmSpec.getCpus(), vmSpec.getMaxSpeed().intValue(), getReservedCpuMHZ(vmSpec), - vmSpec.getLimitCpuUse(), (int) (vmSpec.getMaxRam() / ResourceType.bytesToMiB), getReservedMemoryMb(vmSpec), guestOsId, rootDiskDataStoreDetails.first(), false, - controllerInfo, systemVm)) { - throw new Exception("Failed to create VM. vmName: " + vmInternalCSName); } } @@ -1945,9 +1910,9 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } } - // The number of disks changed must be 0 for install as is, as the VM is a clone - //int disksChanges = !installAsIs ? disks.length : 0; - int totalChangeDevices = disks.length + nics.length; + // The number of disks changed must be 0 for install as is, as the VM is a clone from the template as-is + int disksChanges = !deployAsIs ? disks.length : 0; + int totalChangeDevices = disksChanges + nics.length; int hackDeviceCount = 0; if (diskInfoBuilder != null) { hackDeviceCount += diskInfoBuilder.getDiskCount(); @@ -1964,7 +1929,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa if (vmSpec.getType() != VirtualMachine.Type.User) { // system VM needs a patch ISO totalChangeDevices++; - } else { + } else if (!deployAsIs) { volIso = getIsoDiskTO(disks); if (volIso == null) totalChangeDevices++; @@ -1972,246 +1937,249 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa VirtualMachineConfigSpec vmConfigSpec = new VirtualMachineConfigSpec(); - VmwareHelper.setBasicVmConfig(vmConfigSpec, vmSpec.getCpus(), vmSpec.getMaxSpeed(), getReservedCpuMHZ(vmSpec), (int) (vmSpec.getMaxRam() / (1024 * 1024)), - getReservedMemoryMb(vmSpec), guestOsId, vmSpec.getLimitCpuUse()); - - // Check for multi-cores per socket settings - int numCoresPerSocket = 1; - String coresPerSocket = vmSpec.getDetails().get(VmDetailConstants.CPU_CORE_PER_SOCKET); - if (coresPerSocket != null) { - String apiVersion = HypervisorHostHelper.getVcenterApiVersion(vmMo.getContext()); - // Property 'numCoresPerSocket' is supported since vSphere API 5.0 - if (apiVersion.compareTo("5.0") >= 0) { - numCoresPerSocket = NumbersUtil.parseInt(coresPerSocket, 1); - vmConfigSpec.setNumCoresPerSocket(numCoresPerSocket); - } - } - - // Check for hotadd settings - vmConfigSpec.setMemoryHotAddEnabled(vmMo.isMemoryHotAddSupported(guestOsId)); - - String hostApiVersion = ((HostMO) hyperHost).getHostAboutInfo().getApiVersion(); - if (numCoresPerSocket > 1 && hostApiVersion.compareTo("5.0") < 0) { - s_logger.warn("Dynamic scaling of CPU is not supported for Virtual Machines with multi-core vCPUs in case of ESXi hosts 4.1 and prior. Hence CpuHotAdd will not be" - + " enabled for Virtual Machine: " + vmInternalCSName); - vmConfigSpec.setCpuHotAddEnabled(false); - } else { - vmConfigSpec.setCpuHotAddEnabled(vmMo.isCpuHotAddSupported(guestOsId)); - } - - configNestedHVSupport(vmMo, vmSpec, vmConfigSpec); - - VirtualDeviceConfigSpec[] deviceConfigSpecArray = new VirtualDeviceConfigSpec[totalChangeDevices]; int i = 0; int ideUnitNumber = 0; int scsiUnitNumber = 0; int ideControllerKey = vmMo.getIDEDeviceControllerKey(); int scsiControllerKey = vmMo.getScsiDeviceControllerKeyNoException(); - int controllerKey; + VirtualDeviceConfigSpec[] deviceConfigSpecArray = new VirtualDeviceConfigSpec[totalChangeDevices]; + DiskTO[] sortedDisks = sortVolumesByDeviceId(disks); - // - // Setup ISO device - // + VmwareHelper.setBasicVmConfig(vmConfigSpec, vmSpec.getCpus(), vmSpec.getMaxSpeed(), getReservedCpuMHZ(vmSpec), (int) (vmSpec.getMaxRam() / (1024 * 1024)), + getReservedMemoryMb(vmSpec), guestOsId, vmSpec.getLimitCpuUse(), deployAsIs); - // prepare systemvm patch ISO - if (vmSpec.getType() != VirtualMachine.Type.User) { - // attach ISO (for patching of system VM) - Pair secStoreUrlAndId = mgr.getSecondaryStorageStoreUrlAndId(Long.parseLong(_dcId)); - String secStoreUrl = secStoreUrlAndId.first(); - Long secStoreId = secStoreUrlAndId.second(); - if (secStoreUrl == null) { - String msg = "secondary storage for dc " + _dcId + " is not ready yet?"; - throw new Exception(msg); - } - mgr.prepareSecondaryStorageStore(secStoreUrl, secStoreId); - - ManagedObjectReference morSecDs = prepareSecondaryDatastoreOnHost(secStoreUrl); - if (morSecDs == null) { - String msg = "Failed to prepare secondary storage on host, secondary store url: " + secStoreUrl; - throw new Exception(msg); - } - DatastoreMO secDsMo = new DatastoreMO(hyperHost.getContext(), morSecDs); - - deviceConfigSpecArray[i] = new VirtualDeviceConfigSpec(); - Pair isoInfo = VmwareHelper.prepareIsoDevice(vmMo, - String.format("[%s] systemvm/%s", secDsMo.getName(), mgr.getSystemVMIsoFileNameOnDatastore()), secDsMo.getMor(), true, true, ideUnitNumber++, i + 1); - deviceConfigSpecArray[i].setDevice(isoInfo.first()); - if (isoInfo.second()) { - if (s_logger.isDebugEnabled()) - s_logger.debug("Prepare ISO volume at new device " + _gson.toJson(isoInfo.first())); - deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.ADD); - } else { - if (s_logger.isDebugEnabled()) - s_logger.debug("Prepare ISO volume at existing device " + _gson.toJson(isoInfo.first())); - deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.EDIT); - } - i++; - } else { - // Note: we will always plug a CDROM device - if (volIso != null) { - for (DiskTO vol : disks) { - if (vol.getType() == Volume.Type.ISO) { - - TemplateObjectTO iso = (TemplateObjectTO) vol.getData(); - - if (iso.getPath() != null && !iso.getPath().isEmpty()) { - DataStoreTO imageStore = iso.getDataStore(); - if (!(imageStore instanceof NfsTO)) { - s_logger.debug("unsupported protocol"); - throw new Exception("unsupported protocol"); - } - NfsTO nfsImageStore = (NfsTO) imageStore; - String isoPath = nfsImageStore.getUrl() + File.separator + iso.getPath(); - Pair isoDatastoreInfo = getIsoDatastoreInfo(hyperHost, isoPath); - assert (isoDatastoreInfo != null); - assert (isoDatastoreInfo.second() != null); - - deviceConfigSpecArray[i] = new VirtualDeviceConfigSpec(); - Pair isoInfo = - VmwareHelper.prepareIsoDevice(vmMo, isoDatastoreInfo.first(), isoDatastoreInfo.second(), true, true, ideUnitNumber++, i + 1); - deviceConfigSpecArray[i].setDevice(isoInfo.first()); - if (isoInfo.second()) { - if (s_logger.isDebugEnabled()) - s_logger.debug("Prepare ISO volume at new device " + _gson.toJson(isoInfo.first())); - deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.ADD); - } else { - if (s_logger.isDebugEnabled()) - s_logger.debug("Prepare ISO volume at existing device " + _gson.toJson(isoInfo.first())); - deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.EDIT); - } - } - i++; - } + if (!deployAsIs) { + // Check for multi-cores per socket settings + int numCoresPerSocket = 1; + String coresPerSocket = vmSpec.getDetails().get(VmDetailConstants.CPU_CORE_PER_SOCKET); + if (coresPerSocket != null) { + String apiVersion = HypervisorHostHelper.getVcenterApiVersion(vmMo.getContext()); + // Property 'numCoresPerSocket' is supported since vSphere API 5.0 + if (apiVersion.compareTo("5.0") >= 0) { + numCoresPerSocket = NumbersUtil.parseInt(coresPerSocket, 1); + vmConfigSpec.setNumCoresPerSocket(numCoresPerSocket); } + } + + // Check for hotadd settings + vmConfigSpec.setMemoryHotAddEnabled(vmMo.isMemoryHotAddSupported(guestOsId)); + + String hostApiVersion = ((HostMO) hyperHost).getHostAboutInfo().getApiVersion(); + if (numCoresPerSocket > 1 && hostApiVersion.compareTo("5.0") < 0) { + s_logger.warn("Dynamic scaling of CPU is not supported for Virtual Machines with multi-core vCPUs in case of ESXi hosts 4.1 and prior. Hence CpuHotAdd will not be" + + " enabled for Virtual Machine: " + vmInternalCSName); + vmConfigSpec.setCpuHotAddEnabled(false); } else { + vmConfigSpec.setCpuHotAddEnabled(vmMo.isCpuHotAddSupported(guestOsId)); + } + + configNestedHVSupport(vmMo, vmSpec, vmConfigSpec); + + int controllerKey; + + // + // Setup ISO device + // + + // prepare systemvm patch ISO + if (vmSpec.getType() != VirtualMachine.Type.User) { + // attach ISO (for patching of system VM) + Pair secStoreUrlAndId = mgr.getSecondaryStorageStoreUrlAndId(Long.parseLong(_dcId)); + String secStoreUrl = secStoreUrlAndId.first(); + Long secStoreId = secStoreUrlAndId.second(); + if (secStoreUrl == null) { + String msg = "secondary storage for dc " + _dcId + " is not ready yet?"; + throw new Exception(msg); + } + mgr.prepareSecondaryStorageStore(secStoreUrl, secStoreId); + + ManagedObjectReference morSecDs = prepareSecondaryDatastoreOnHost(secStoreUrl); + if (morSecDs == null) { + String msg = "Failed to prepare secondary storage on host, secondary store url: " + secStoreUrl; + throw new Exception(msg); + } + DatastoreMO secDsMo = new DatastoreMO(hyperHost.getContext(), morSecDs); + deviceConfigSpecArray[i] = new VirtualDeviceConfigSpec(); - Pair isoInfo = VmwareHelper.prepareIsoDevice(vmMo, null, null, true, true, ideUnitNumber++, i + 1); + Pair isoInfo = VmwareHelper.prepareIsoDevice(vmMo, + String.format("[%s] systemvm/%s", secDsMo.getName(), mgr.getSystemVMIsoFileNameOnDatastore()), secDsMo.getMor(), true, true, ideUnitNumber++, i + 1); deviceConfigSpecArray[i].setDevice(isoInfo.first()); if (isoInfo.second()) { if (s_logger.isDebugEnabled()) - s_logger.debug("Prepare ISO volume at existing device " + _gson.toJson(isoInfo.first())); - + s_logger.debug("Prepare ISO volume at new device " + _gson.toJson(isoInfo.first())); deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.ADD); } else { if (s_logger.isDebugEnabled()) s_logger.debug("Prepare ISO volume at existing device " + _gson.toJson(isoInfo.first())); - deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.EDIT); } i++; + } else { + // Note: we will always plug a CDROM device + if (volIso != null) { + for (DiskTO vol : disks) { + if (vol.getType() == Volume.Type.ISO) { + + TemplateObjectTO iso = (TemplateObjectTO) vol.getData(); + + if (iso.getPath() != null && !iso.getPath().isEmpty()) { + DataStoreTO imageStore = iso.getDataStore(); + if (!(imageStore instanceof NfsTO)) { + s_logger.debug("unsupported protocol"); + throw new Exception("unsupported protocol"); + } + NfsTO nfsImageStore = (NfsTO) imageStore; + String isoPath = nfsImageStore.getUrl() + File.separator + iso.getPath(); + Pair isoDatastoreInfo = getIsoDatastoreInfo(hyperHost, isoPath); + assert (isoDatastoreInfo != null); + assert (isoDatastoreInfo.second() != null); + + deviceConfigSpecArray[i] = new VirtualDeviceConfigSpec(); + Pair isoInfo = + VmwareHelper.prepareIsoDevice(vmMo, isoDatastoreInfo.first(), isoDatastoreInfo.second(), true, true, ideUnitNumber++, i + 1); + deviceConfigSpecArray[i].setDevice(isoInfo.first()); + if (isoInfo.second()) { + if (s_logger.isDebugEnabled()) + s_logger.debug("Prepare ISO volume at new device " + _gson.toJson(isoInfo.first())); + deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.ADD); + } else { + if (s_logger.isDebugEnabled()) + s_logger.debug("Prepare ISO volume at existing device " + _gson.toJson(isoInfo.first())); + deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.EDIT); + } + } + i++; + } + } + } else { + deviceConfigSpecArray[i] = new VirtualDeviceConfigSpec(); + Pair isoInfo = VmwareHelper.prepareIsoDevice(vmMo, null, null, true, true, ideUnitNumber++, i + 1); + deviceConfigSpecArray[i].setDevice(isoInfo.first()); + if (isoInfo.second()) { + if (s_logger.isDebugEnabled()) + s_logger.debug("Prepare ISO volume at existing device " + _gson.toJson(isoInfo.first())); + + deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.ADD); + } else { + if (s_logger.isDebugEnabled()) + s_logger.debug("Prepare ISO volume at existing device " + _gson.toJson(isoInfo.first())); + + deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.EDIT); + } + i++; + } } - } - // - // Setup ROOT/DATA disk devices - // - DiskTO[] sortedDisks = sortVolumesByDeviceId(disks); - for (DiskTO vol : sortedDisks) { - if (vol.getType() == Volume.Type.ISO) - continue; + // + // Setup ROOT/DATA disk devices + // + for (DiskTO vol : sortedDisks) { + if (vol.getType() == Volume.Type.ISO) + continue; - VirtualMachineDiskInfo matchingExistingDisk = getMatchingExistingDisk(diskInfoBuilder, vol, hyperHost, context); - controllerKey = getDiskController(matchingExistingDisk, vol, vmSpec, ideControllerKey, scsiControllerKey); - String diskController = getDiskController(vmMo, matchingExistingDisk, vol, new Pair(rootDiskController, dataDiskController)); + VirtualMachineDiskInfo matchingExistingDisk = getMatchingExistingDisk(diskInfoBuilder, vol, hyperHost, context); + controllerKey = getDiskController(matchingExistingDisk, vol, vmSpec, ideControllerKey, scsiControllerKey); + String diskController = getDiskController(vmMo, matchingExistingDisk, vol, controllerInfo); - if (DiskControllerType.getType(diskController) == DiskControllerType.osdefault) { - diskController = vmMo.getRecommendedDiskController(null); - } - if (DiskControllerType.getType(diskController) == DiskControllerType.ide) { - controllerKey = vmMo.getIDEControllerKey(ideUnitNumber); - if (vol.getType() == Volume.Type.DATADISK) { - // Could be result of flip due to user configured setting or "osdefault" for data disks - // Ensure maximum of 2 data volumes over IDE controller, 3 includeing root volume - if (vmMo.getNumberOfVirtualDisks() > 3) { - throw new CloudRuntimeException("Found more than 3 virtual disks attached to this VM [" + vmMo.getVmName() + "]. Unable to implement the disks over " - + diskController + " controller, as maximum number of devices supported over IDE controller is 4 includeing CDROM device."); + if (DiskControllerType.getType(diskController) == DiskControllerType.osdefault) { + diskController = vmMo.getRecommendedDiskController(null); + } + if (DiskControllerType.getType(diskController) == DiskControllerType.ide) { + controllerKey = vmMo.getIDEControllerKey(ideUnitNumber); + if (vol.getType() == Volume.Type.DATADISK) { + // Could be result of flip due to user configured setting or "osdefault" for data disks + // Ensure maximum of 2 data volumes over IDE controller, 3 includeing root volume + if (vmMo.getNumberOfVirtualDisks() > 3) { + throw new CloudRuntimeException("Found more than 3 virtual disks attached to this VM [" + vmMo.getVmName() + "]. Unable to implement the disks over " + + diskController + " controller, as maximum number of devices supported over IDE controller is 4 includeing CDROM device."); + } + } + } else { + if (VmwareHelper.isReservedScsiDeviceNumber(scsiUnitNumber)) { + scsiUnitNumber++; + } + + controllerKey = vmMo.getScsiDiskControllerKeyNoException(diskController, scsiUnitNumber); + if (controllerKey == -1) { + // This may happen for ROOT legacy VMs which doesn't have recommended disk controller when global configuration parameter 'vmware.root.disk.controller' is set to "osdefault" + // Retrieve existing controller and use. + Ternary vmScsiControllerInfo = vmMo.getScsiControllerInfo(); + DiskControllerType existingControllerType = vmScsiControllerInfo.third(); + controllerKey = vmMo.getScsiDiskControllerKeyNoException(existingControllerType.toString(), scsiUnitNumber); } } - } else { - if (VmwareHelper.isReservedScsiDeviceNumber(scsiUnitNumber)) { - scsiUnitNumber++; - } + if (!hasSnapshot) { + deviceConfigSpecArray[i] = new VirtualDeviceConfigSpec(); - controllerKey = vmMo.getScsiDiskControllerKeyNoException(diskController, scsiUnitNumber); - if (controllerKey == -1) { - // This may happen for ROOT legacy VMs which doesn't have recommended disk controller when global configuration parameter 'vmware.root.disk.controller' is set to "osdefault" - // Retrieve existing controller and use. - Ternary vmScsiControllerInfo = vmMo.getScsiControllerInfo(); - DiskControllerType existingControllerType = vmScsiControllerInfo.third(); - controllerKey = vmMo.getScsiDiskControllerKeyNoException(existingControllerType.toString(), scsiUnitNumber); - } - } - if (!hasSnapshot) { - deviceConfigSpecArray[i] = new VirtualDeviceConfigSpec(); + VolumeObjectTO volumeTO = (VolumeObjectTO) vol.getData(); + DataStoreTO primaryStore = volumeTO.getDataStore(); + Map details = vol.getDetails(); + boolean managed = false; + String iScsiName = null; - VolumeObjectTO volumeTO = (VolumeObjectTO) vol.getData(); - DataStoreTO primaryStore = volumeTO.getDataStore(); - Map details = vol.getDetails(); - boolean managed = false; - String iScsiName = null; + if (details != null) { + managed = Boolean.parseBoolean(details.get(DiskTO.MANAGED)); + iScsiName = details.get(DiskTO.IQN); + } - if (details != null) { - managed = Boolean.parseBoolean(details.get(DiskTO.MANAGED)); - iScsiName = details.get(DiskTO.IQN); - } + // if the storage is managed, iScsiName should not be null + String datastoreName = managed ? VmwareResource.getDatastoreName(iScsiName) : primaryStore.getUuid(); + Pair volumeDsDetails = dataStoresDetails.get(datastoreName); - // if the storage is managed, iScsiName should not be null - String datastoreName = managed ? VmwareResource.getDatastoreName(iScsiName) : primaryStore.getUuid(); - Pair volumeDsDetails = dataStoresDetails.get(datastoreName); + assert (volumeDsDetails != null); - assert (volumeDsDetails != null); + String[] diskChain = syncDiskChain(dcMo, vmMo, vmSpec, vol, matchingExistingDisk, dataStoresDetails); - String[] diskChain = syncDiskChain(dcMo, vmMo, vmSpec, vol, matchingExistingDisk, dataStoresDetails); + int deviceNumber = -1; + if (controllerKey == vmMo.getIDEControllerKey(ideUnitNumber)) { + deviceNumber = ideUnitNumber % VmwareHelper.MAX_ALLOWED_DEVICES_IDE_CONTROLLER; + ideUnitNumber++; + } else { + deviceNumber = scsiUnitNumber % VmwareHelper.MAX_ALLOWED_DEVICES_SCSI_CONTROLLER; + scsiUnitNumber++; + } - int deviceNumber = -1; - if (controllerKey == vmMo.getIDEControllerKey(ideUnitNumber)) { - deviceNumber = ideUnitNumber % VmwareHelper.MAX_ALLOWED_DEVICES_IDE_CONTROLLER; - ideUnitNumber++; + VirtualDevice device = VmwareHelper.prepareDiskDevice(vmMo, null, controllerKey, diskChain, volumeDsDetails.first(), deviceNumber, i + 1); + + if (vol.getType() == Volume.Type.ROOT) + rootDiskTO = vol; + deviceConfigSpecArray[i].setDevice(device); + deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.ADD); + + if (s_logger.isDebugEnabled()) + s_logger.debug("Prepare volume at new device " + _gson.toJson(device)); + + i++; } else { - deviceNumber = scsiUnitNumber % VmwareHelper.MAX_ALLOWED_DEVICES_SCSI_CONTROLLER; - scsiUnitNumber++; + if (controllerKey == vmMo.getIDEControllerKey(ideUnitNumber)) + ideUnitNumber++; + else + scsiUnitNumber++; } - - VirtualDevice device = VmwareHelper.prepareDiskDevice(vmMo, null, controllerKey, diskChain, volumeDsDetails.first(), deviceNumber, i + 1); - - if (vol.getType() == Volume.Type.ROOT) - rootDiskTO = vol; - deviceConfigSpecArray[i].setDevice(device); - deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.ADD); - - if (s_logger.isDebugEnabled()) - s_logger.debug("Prepare volume at new device " + _gson.toJson(device)); - - i++; - } else { - if (controllerKey == vmMo.getIDEControllerKey(ideUnitNumber)) - ideUnitNumber++; - else - scsiUnitNumber++; } - } - // - // Setup USB devices - // - if (guestOsId.startsWith("darwin")) { //Mac OS - VirtualDevice[] devices = vmMo.getMatchedDevices(new Class[]{VirtualUSBController.class}); - if (devices.length == 0) { - s_logger.debug("No USB Controller device on VM Start. Add USB Controller device for Mac OS VM " + vmInternalCSName); + // + // Setup USB devices + // + if (guestOsId.startsWith("darwin")) { //Mac OS + VirtualDevice[] devices = vmMo.getMatchedDevices(new Class[]{VirtualUSBController.class}); + if (devices.length == 0) { + s_logger.debug("No USB Controller device on VM Start. Add USB Controller device for Mac OS VM " + vmInternalCSName); - //For Mac OS X systems, the EHCI+UHCI controller is enabled by default and is required for USB mouse and keyboard access. - VirtualDevice usbControllerDevice = VmwareHelper.prepareUSBControllerDevice(); - deviceConfigSpecArray[i] = new VirtualDeviceConfigSpec(); - deviceConfigSpecArray[i].setDevice(usbControllerDevice); - deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.ADD); + //For Mac OS X systems, the EHCI+UHCI controller is enabled by default and is required for USB mouse and keyboard access. + VirtualDevice usbControllerDevice = VmwareHelper.prepareUSBControllerDevice(); + deviceConfigSpecArray[i] = new VirtualDeviceConfigSpec(); + deviceConfigSpecArray[i].setDevice(usbControllerDevice); + deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.ADD); - if (s_logger.isDebugEnabled()) - s_logger.debug("Prepare USB controller at new device " + _gson.toJson(deviceConfigSpecArray[i])); + if (s_logger.isDebugEnabled()) + s_logger.debug("Prepare USB controller at new device " + _gson.toJson(deviceConfigSpecArray[i])); - i++; - } else { - s_logger.debug("USB Controller device exists on VM Start for Mac OS VM " + vmInternalCSName); + i++; + } else { + s_logger.debug("USB Controller device exists on VM Start for Mac OS VM " + vmInternalCSName); + } } } @@ -2318,33 +2286,32 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa // pass boot arguments through machine.id & perform customized options to VMX ArrayList extraOptions = new ArrayList(); configBasicExtraOption(extraOptions, vmSpec); - configNvpExtraOption(extraOptions, vmSpec, nicUuidToDvSwitchUuid); - configCustomExtraOption(extraOptions, vmSpec); - // config for NCC - VirtualMachine.Type vmType = cmd.getVirtualMachine().getType(); - if (vmType.equals(VirtualMachine.Type.NetScalerVm)) { - NicTO mgmtNic = vmSpec.getNics()[0]; - OptionValue option = new OptionValue(); - option.setKey("machine.id"); - option.setValue("ip=" + mgmtNic.getIp() + "&netmask=" + mgmtNic.getNetmask() + "&gateway=" + mgmtNic.getGateway()); - extraOptions.add(option); + if (deployAsIs) { + setDeployAsIsProperties(vmMo, deployAsIsInfo, vmConfigSpec); + configureVNC(vmSpec, extraOptions, vmConfigSpec, hyperHost, vmInternalCSName); + } else { + configNvpExtraOption(extraOptions, vmSpec, nicUuidToDvSwitchUuid); + configCustomExtraOption(extraOptions, vmSpec); + + // config for NCC + VirtualMachine.Type vmType = cmd.getVirtualMachine().getType(); + if (vmType.equals(VirtualMachine.Type.NetScalerVm)) { + NicTO mgmtNic = vmSpec.getNics()[0]; + OptionValue option = new OptionValue(); + option.setKey("machine.id"); + option.setValue("ip=" + mgmtNic.getIp() + "&netmask=" + mgmtNic.getNetmask() + "&gateway=" + mgmtNic.getGateway()); + extraOptions.add(option); + } + + configureVNC(vmSpec, extraOptions, vmConfigSpec, hyperHost, vmInternalCSName); + + // config video card + configureVideoCard(vmMo, vmSpec, vmConfigSpec); + + setBootOptions(vmSpec, bootMode, vmConfigSpec); } - // config VNC - String keyboardLayout = null; - if (vmSpec.getDetails() != null) - keyboardLayout = vmSpec.getDetails().get(VmDetailConstants.KEYBOARD); - vmConfigSpec.getExtraConfig() - .addAll(Arrays.asList(configureVnc(extraOptions.toArray(new OptionValue[0]), hyperHost, vmInternalCSName, vmSpec.getVncPassword(), keyboardLayout))); - - // config video card - configureVideoCard(vmMo, vmSpec, vmConfigSpec); - - setDeployAsIsProperties(vmMo, deployAsIsInfo, vmConfigSpec); - - setBootOptions(vmSpec, bootMode, vmConfigSpec); - // // Configure VM // @@ -2358,7 +2325,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa // Resizing root disk only when explicit requested by user final Map vmDetails = cmd.getVirtualMachine().getDetails(); - if (rootDiskTO != null && !hasSnapshot && (vmDetails != null && vmDetails.containsKey(ApiConstants.ROOT_DISK_SIZE))) { + if (!deployAsIs && rootDiskTO != null && !hasSnapshot && (vmDetails != null && vmDetails.containsKey(ApiConstants.ROOT_DISK_SIZE))) { resizeRootDiskOnVMStart(vmMo, rootDiskTO, hyperHost, context); } @@ -2432,6 +2399,85 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } } + /** + * Configure VNC + */ + private void configureVNC(VirtualMachineTO vmSpec, ArrayList extraOptions, VirtualMachineConfigSpec vmConfigSpec, VmwareHypervisorHost hyperHost, String vmInternalCSName) throws Exception { + String keyboardLayout = null; + if (vmSpec.getDetails() != null) + keyboardLayout = vmSpec.getDetails().get(VmDetailConstants.KEYBOARD); + vmConfigSpec.getExtraConfig() + .addAll(Arrays.asList(configureVnc(extraOptions.toArray(new OptionValue[0]), hyperHost, vmInternalCSName, vmSpec.getVncPassword(), keyboardLayout))); + + } + + private void ensureDiskControllersInternal(VirtualMachineMO vmMo, Boolean systemVm, + Pair controllerInfo, + DiskControllerType systemVmScsiControllerType, + int numScsiControllerForSystemVm, + int firstScsiControllerBusNum, boolean deployAsIs) throws Exception { + if (systemVm) { + ensureScsiDiskControllers(vmMo, systemVmScsiControllerType.toString(), numScsiControllerForSystemVm, firstScsiControllerBusNum); + } else if (!deployAsIs) { + ensureDiskControllers(vmMo, controllerInfo); + } + } + + private void tearDownVmDevices(VirtualMachineMO vmMo, boolean hasSnapshot, boolean deployAsIs) throws Exception { + if (deployAsIs) { + vmMo.tearDownDevices(new Class[]{VirtualEthernetCard.class}); + } else if (!hasSnapshot) { + vmMo.tearDownDevices(new Class[]{VirtualDisk.class, VirtualEthernetCard.class}); + } else { + vmMo.tearDownDevices(new Class[]{VirtualEthernetCard.class}); + } + } + + private String getGuestOsIdFromVmSpec(VirtualMachineTO vmSpec, boolean deployAsIs) { + String guestOsId = null; + if (!deployAsIs) { + guestOsId = translateGuestOsIdentifier(vmSpec.getArch(), vmSpec.getOs(), vmSpec.getPlatformEmulator()).value(); + } + return guestOsId; + } + + private Pair getControllerInfoFromVmSpec(VirtualMachineTO vmSpec, boolean deployAsIs) throws CloudRuntimeException { + String dataDiskController = vmSpec.getDetails().get(VmDetailConstants.DATA_DISK_CONTROLLER); + String rootDiskController = vmSpec.getDetails().get(VmDetailConstants.ROOT_DISK_CONTROLLER); + + // If root disk controller is scsi, then data disk controller would also be scsi instead of using 'osdefault' + // This helps avoid mix of different scsi subtype controllers in instance. + if (DiskControllerType.osdefault == DiskControllerType.getType(dataDiskController) && DiskControllerType.lsilogic == DiskControllerType.getType(rootDiskController)) { + dataDiskController = DiskControllerType.scsi.toString(); + } + + // Validate the controller types + dataDiskController = DiskControllerType.getType(dataDiskController).toString(); + rootDiskController = DiskControllerType.getType(rootDiskController).toString(); + + if (DiskControllerType.getType(rootDiskController) == DiskControllerType.none) { + throw new CloudRuntimeException("Invalid root disk controller detected : " + rootDiskController); + } + if (DiskControllerType.getType(dataDiskController) == DiskControllerType.none) { + throw new CloudRuntimeException("Invalid data disk controller detected : " + dataDiskController); + } + + return new Pair(rootDiskController, dataDiskController); + } + + private String getBootModeFromVmSpec(VirtualMachineTO vmSpec, boolean deployAsIs) { + String bootMode = null; + if (!deployAsIs) { + if (vmSpec.getDetails().containsKey(VmDetailConstants.BOOT_MODE)) { + bootMode = vmSpec.getDetails().get(VmDetailConstants.BOOT_MODE); + } + if (null == bootMode) { + bootMode = ApiConstants.BootType.BIOS.toString(); + } + } + return bootMode; + } + /** * Set OVF properties (if available) */ diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java index 1ac9636e0e7..37d97aeaf31 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/HypervisorHostHelper.java @@ -1479,7 +1479,7 @@ public class HypervisorHostHelper { if (vmInternalCSName == null) vmInternalCSName = vmName; - VmwareHelper.setBasicVmConfig(vmConfig, cpuCount, cpuSpeedMHz, cpuReservedMHz, memoryMB, memoryReserveMB, guestOsIdentifier, limitCpuUse); + VmwareHelper.setBasicVmConfig(vmConfig, cpuCount, cpuSpeedMHz, cpuReservedMHz, memoryMB, memoryReserveMB, guestOsIdentifier, limitCpuUse, false); String recommendedController = host.getRecommendedDiskController(guestOsIdentifier); String newRootDiskController = controllerInfo.first(); diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java index e00bc237018..424027ba712 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareHelper.java @@ -534,7 +534,7 @@ public class VmwareHelper { } public static void setBasicVmConfig(VirtualMachineConfigSpec vmConfig, int cpuCount, int cpuSpeedMHz, int cpuReservedMhz, int memoryMB, int memoryReserveMB, - String guestOsIdentifier, boolean limitCpuUse) { + String guestOsIdentifier, boolean limitCpuUse, boolean deployAsIs) { // VM config basics vmConfig.setMemoryMB((long)memoryMB); @@ -560,7 +560,11 @@ public class VmwareHelper { memInfo.setReservation((long)memoryReserveMB); vmConfig.setMemoryAllocation(memInfo); - vmConfig.setGuestId(guestOsIdentifier); + if (!deployAsIs) { + // Deploy as-is uses the cloned VM guest OS + vmConfig.setGuestId(guestOsIdentifier); + } + } public static VirtualDevice prepareUSBControllerDevice() { From a43d0a6085fe9bbcb84fdae92484355910060da0 Mon Sep 17 00:00:00 2001 From: nvazquez Date: Sun, 20 Sep 2020 01:43:04 -0300 Subject: [PATCH 096/164] Fix VM power sync NPE on system VMs --- .../main/java/com/cloud/vm/VirtualMachineManagerImpl.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 7ed090db9ff..013eb411ad8 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -1666,9 +1666,11 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } guru.finalizeStop(profile, answer); - final UserVmVO userVm = _userVmDao.findById(vm.getId()); - userVm.setPowerState(PowerState.PowerOff); - _userVmDao.update(userVm.getId(), userVm); + if (vm.getType() == VirtualMachine.Type.User) { + final UserVmVO userVm = _userVmDao.findById(vm.getId()); + userVm.setPowerState(PowerState.PowerOff); + _userVmDao.update(userVm.getId(), userVm); + } } else { s_logger.error("Invalid answer received in response to a StopCommand for " + vm.getInstanceName()); return false; From 0f5a6ee589022f513ea494614e775ebc46267ef0 Mon Sep 17 00:00:00 2001 From: nvazquez Date: Sun, 20 Sep 2020 15:56:59 -0300 Subject: [PATCH 097/164] Remove deploy as-is details when removing templates or vms --- .../java/com/cloud/vm/VirtualMachineManagerImpl.java | 10 ++++++++++ .../com/cloud/template/HypervisorTemplateAdapter.java | 8 ++++++++ 2 files changed, 18 insertions(+) diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 013eb411ad8..8a31556a221 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -40,6 +40,7 @@ import java.util.concurrent.TimeUnit; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.deployasis.dao.UserVmDeployAsIsDetailsDao; import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.command.admin.vm.MigrateVMCmd; @@ -344,6 +345,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac private NetworkDetailsDao networkDetailsDao; @Inject private SecurityGroupManager _securityGroupManager; + @Inject + private UserVmDeployAsIsDetailsDao userVmDeployAsIsDetailsDao; VmWorkJobHandlerProxy _jobHandlerProxy = new VmWorkJobHandlerProxy(this); @@ -594,6 +597,13 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac //remove the overcommit details from the uservm details userVmDetailsDao.removeDetails(vm.getId()); + // Remove details if VM deploy as-is + long templateId = vm.getTemplateId(); + VMTemplateVO template = _templateDao.findById(templateId); + if (template != null && template.isDeployAsIs()) { + userVmDeployAsIsDetailsDao.removeDetails(vm.getId()); + } + // send hypervisor-dependent commands before removing final List finalizeExpungeCommands = hvGuru.finalizeExpunge(vm); if (finalizeExpungeCommands != null && finalizeExpungeCommands.size() > 0) { diff --git a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java index 95fe38b7189..18d37d0b8ad 100644 --- a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java +++ b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java @@ -26,6 +26,7 @@ import java.util.concurrent.ExecutionException; import javax.inject.Inject; import com.cloud.configuration.Config; +import com.cloud.deployasis.dao.TemplateDeployAsIsDetailsDao; import com.cloud.storage.dao.VMTemplateDetailsDao; import com.cloud.utils.db.Transaction; import com.cloud.utils.db.TransactionCallback; @@ -129,6 +130,8 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { VMTemplateDao templateDao; @Inject private VMTemplateDetailsDao templateDetailsDao; + @Inject + private TemplateDeployAsIsDetailsDao templateDeployAsIsDetailsDao; @Override public String getName() { @@ -598,6 +601,11 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { // Remove template details templateDetailsDao.removeDetails(template.getId()); + // Remove deploy-as-is details + if (template.isDeployAsIs()) { + templateDeployAsIsDetailsDao.removeDetails(template.getId()); + } + } return success; From edfbed34adb978e3798e2b84163d2a5b9b5e959e Mon Sep 17 00:00:00 2001 From: nvazquez Date: Mon, 21 Sep 2020 00:12:28 -0300 Subject: [PATCH 098/164] Use network adapter from OVF on deploy-as-is --- .../cloud/agent/api/to/DeployAsIsInfoTO.java | 8 +++++++- .../image/deployasis/DeployAsIsHelper.java | 3 +++ .../deployasis/DeployAsIsHelperImpl.java | 20 +++++++++++++++++++ .../hypervisor/guru/VmwareVmImplementer.java | 3 ++- .../vmware/resource/VmwareResource.java | 12 ++++++++--- 5 files changed, 41 insertions(+), 5 deletions(-) diff --git a/api/src/main/java/com/cloud/agent/api/to/DeployAsIsInfoTO.java b/api/src/main/java/com/cloud/agent/api/to/DeployAsIsInfoTO.java index cc0e1d01d03..d82a2975eaf 100644 --- a/api/src/main/java/com/cloud/agent/api/to/DeployAsIsInfoTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/DeployAsIsInfoTO.java @@ -27,14 +27,17 @@ public class DeployAsIsInfoTO { private String destStoragePool; @LogLevel(LogLevel.Log4jLevel.Off) private Map properties = new HashMap<>(); + private Map nicAdapterMap = new HashMap(); public DeployAsIsInfoTO() { } - public DeployAsIsInfoTO(String templatePath, String destStoragePool, Map properties) { + public DeployAsIsInfoTO(String templatePath, String destStoragePool, Map properties, + Map nicAdapterMap) { this.templatePath = templatePath; this.destStoragePool = destStoragePool; this.properties = properties; + this.nicAdapterMap = nicAdapterMap; } public String getTemplatePath() { @@ -49,4 +52,7 @@ public class DeployAsIsInfoTO { return destStoragePool; } + public Map getNicAdapterMap() { + return nicAdapterMap; + } } diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/deployasis/DeployAsIsHelper.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/deployasis/DeployAsIsHelper.java index 574e8cdf7bd..303161c0c59 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/deployasis/DeployAsIsHelper.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/deployasis/DeployAsIsHelper.java @@ -17,6 +17,7 @@ package org.apache.cloudstack.storage.image.deployasis; import com.cloud.agent.api.storage.DownloadAnswer; +import com.cloud.agent.api.to.NicTO; import com.cloud.vm.VirtualMachineProfile; import java.util.Map; @@ -28,4 +29,6 @@ public interface DeployAsIsHelper { String getAllocatedVirtualMachineTemplatePath(VirtualMachineProfile vm, String configuration, String destStoragePool); String getAllocatedVirtualMachineDestinationStoragePool(VirtualMachineProfile vm); + + Map getAllocatedVirtualMachineNicsAdapterMapping(VirtualMachineProfile vm, NicTO[] nics); } diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/deployasis/DeployAsIsHelperImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/deployasis/DeployAsIsHelperImpl.java index fa3f0cbebc4..7e431b1a729 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/deployasis/DeployAsIsHelperImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/deployasis/DeployAsIsHelperImpl.java @@ -22,6 +22,7 @@ import com.cloud.agent.api.storage.DownloadAnswer; import com.cloud.agent.api.to.DataStoreTO; import com.cloud.agent.api.to.DataTO; import com.cloud.agent.api.to.DiskTO; +import com.cloud.agent.api.to.NicTO; import com.cloud.agent.api.to.deployasis.OVFConfigurationTO; import com.cloud.agent.api.to.deployasis.OVFEulaSectionTO; import com.cloud.agent.api.to.deployasis.OVFPropertyTO; @@ -45,6 +46,7 @@ import com.cloud.agent.api.to.deployasis.OVFNetworkTO; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.ArrayUtils; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -146,6 +148,24 @@ public class DeployAsIsHelperImpl implements DeployAsIsHelper { return null; } + @Override + public Map getAllocatedVirtualMachineNicsAdapterMapping(VirtualMachineProfile vm, NicTO[] nics) { + Map map = new HashMap<>(); + List networks = templateDeployAsIsDetailsDao.listNetworkRequirementsByTemplateId(vm.getTemplateId()); + if (ArrayUtils.isNotEmpty(nics)) { + if (nics.length != networks.size()) { + String msg = "Different number of networks provided vs networks defined in deploy-as-is template"; + LOGGER.error(msg); + return map; + } + for (int i = 0; i < nics.length; i++) { + // The nic Adapter is defined on the resource sub type + map.put(nics[i].getDeviceId(), networks.get(i).getResourceSubType()); + } + } + return map; + } + private void persistTemplateDeployAsIsInformationTOList(long templateId, List informationTOList) { for (TemplateDeployAsIsInformationTO informationTO : informationTOList) { diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java index 56a875d8ab2..5c61887176c 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java @@ -190,7 +190,8 @@ class VmwareVmImplementer { Map properties = deployAsIsHelper.getVirtualMachineDeployAsIsProperties(vm); String destStoragePool = deployAsIsHelper.getAllocatedVirtualMachineDestinationStoragePool(vm); String templatePath = deployAsIsHelper.getAllocatedVirtualMachineTemplatePath(vm, configuration, destStoragePool); - DeployAsIsInfoTO info = new DeployAsIsInfoTO(templatePath, destStoragePool, properties); + Map nicsAdapterMapping = deployAsIsHelper.getAllocatedVirtualMachineNicsAdapterMapping(vm, to.getNics()); + DeployAsIsInfoTO info = new DeployAsIsInfoTO(templatePath, destStoragePool, properties, nicsAdapterMapping); to.setDeployAsIsInfo(info); } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index e2db99fea4b..b07e7714646 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -2223,9 +2223,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } } - VirtualEthernetCardType nicDeviceType = VirtualEthernetCardType.valueOf(vmSpec.getDetails().get(VmDetailConstants.NIC_ADAPTER)); - if (s_logger.isDebugEnabled()) - s_logger.debug("VM " + vmInternalCSName + " will be started with NIC device type: " + nicDeviceType); + VirtualEthernetCardType nicDeviceType; NiciraNvpApiVersion.logNiciraApiVersion(); @@ -2233,6 +2231,14 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa for (NicTO nicTo : sortNicsByDeviceId(nics)) { s_logger.info("Prepare NIC device based on NicTO: " + _gson.toJson(nicTo)); + String adapterTypeStr = deployAsIs ? + deployAsIsInfo.getNicAdapterMap().get(nicTo.getDeviceId()) : + vmSpec.getDetails().get(VmDetailConstants.NIC_ADAPTER); + nicDeviceType = VirtualEthernetCardType.valueOf(adapterTypeStr); + + if (s_logger.isDebugEnabled()) { + s_logger.debug("VM " + vmInternalCSName + " will be started with NIC device type: " + nicDeviceType + " on NIC device " + nicTo.getDeviceId()); + } boolean configureVServiceInNexus = (nicTo.getType() == TrafficType.Guest) && (vmSpec.getDetails().containsKey("ConfigureVServiceInNexus")); VirtualMachine.Type vmType = cmd.getVirtualMachine().getType(); Pair networkInfo = prepareNetworkFromNicInfo(vmMo.getRunningHost(), nicTo, configureVServiceInNexus, vmType); From acf872cdabca544bf13e4d9bd7d0a447d52413f8 Mon Sep 17 00:00:00 2001 From: nvazquez Date: Tue, 22 Sep 2020 01:03:19 -0300 Subject: [PATCH 099/164] Relocate deploy-as-is cloned VM if does not end on the host receiving the start command and restore deployment logs --- .../cloud/hypervisor/vmware/resource/VmwareResource.java | 7 ++++++- .../com/cloud/storage/resource/VmwareStorageProcessor.java | 7 +++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index b07e7714646..fc76bc11830 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -1858,7 +1858,12 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } String deployAsIsTemplate = deployAsIsInfo.getTemplatePath(); String destDatastore = deployAsIsInfo.getDestStoragePool(); - vmMo = _storageProcessor.cloneVMFromTemplate(deployAsIsTemplate, vmInternalCSName, destDatastore); + _storageProcessor.cloneVMFromTemplate(hyperHost, deployAsIsTemplate, vmInternalCSName, destDatastore); + vmMo = hyperHost.findVmOnHyperHost(vmInternalCSName); + if (vmMo == null) { + s_logger.info("Cloned deploy-as-is VM " + vmInternalCSName + " is not in this host, relocating it"); + vmMo = takeVmFromOtherHyperHost(hyperHost, vmInternalCSName); + } mapSpecDisksToClonedDisks(vmMo, vmInternalCSName, specDisks); } else { Pair rootDiskDataStoreDetails = null; diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java index fe9262310e0..fab928c0520 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -3729,10 +3729,9 @@ public class VmwareStorageProcessor implements StorageProcessor { /** * Return the cloned VM from the template */ - public VirtualMachineMO cloneVMFromTemplate(String templateName, String cloneName, String templatePrimaryStoreUuid) { + public VirtualMachineMO cloneVMFromTemplate(VmwareHypervisorHost hyperHost, String templateName, String cloneName, String templatePrimaryStoreUuid) { try { - VmwareContext context = hostService.getServiceContext(null); - VmwareHypervisorHost hyperHost = hostService.getHyperHost(context, null); + VmwareContext context = hyperHost.getContext(); DatacenterMO dcMo = new DatacenterMO(context, hyperHost.getHyperHostDatacenter()); VirtualMachineMO templateMo = dcMo.findVm(templateName); if (templateMo == null) { @@ -3744,7 +3743,7 @@ public class VmwareStorageProcessor implements StorageProcessor { if (morDatastore == null) { throw new CloudRuntimeException("Unable to find datastore in vSphere"); } - s_logger.info("Cloning VM " + cloneName + " from template " + templatePrimaryStoreUuid); + s_logger.info("Cloning VM " + cloneName + " from template " + templateName + " into datastore " + templatePrimaryStoreUuid); if (!_fullCloneFlag) { createVMLinkedClone(templateMo, dcMo, cloneName, morDatastore, morPool); } else { From c6c4bfe30ca3026ea1ca7cc32916460d912c0623 Mon Sep 17 00:00:00 2001 From: nvazquez Date: Tue, 22 Sep 2020 21:15:28 -0300 Subject: [PATCH 100/164] Fix deploy as is with datadisks --- .../vmware/resource/VmwareResource.java | 389 +++++++++--------- .../resource/VmwareStorageProcessor.java | 4 +- 2 files changed, 200 insertions(+), 193 deletions(-) diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index fc76bc11830..91bebcabaf9 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -1757,7 +1757,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa String vmNameOnVcenter = names.second(); DiskTO rootDiskTO = null; String bootMode = getBootModeFromVmSpec(vmSpec, deployAsIs); - Pair controllerInfo = getControllerInfoFromVmSpec(vmSpec, deployAsIs); + Pair controllerInfo = getControllerInfoFromVmSpec(vmSpec); Boolean systemVm = vmSpec.getType().isUsedBySystem(); // Thus, vmInternalCSName always holds i-x-y, the cloudstack generated internal VM name. @@ -1915,17 +1915,8 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } } - // The number of disks changed must be 0 for install as is, as the VM is a clone from the template as-is - int disksChanges = !deployAsIs ? disks.length : 0; + int disksChanges = getDisksChangesNumberFromDisksSpec(disks, deployAsIs); int totalChangeDevices = disksChanges + nics.length; - int hackDeviceCount = 0; - if (diskInfoBuilder != null) { - hackDeviceCount += diskInfoBuilder.getDiskCount(); - } - hackDeviceCount += nicDevices == null ? 0 : nicDevices.length; - if (s_logger.isTraceEnabled()) { - s_logger.trace(String.format("current count(s) desired: %d/ found:%d. now adding device to device count for vApp config ISO", totalChangeDevices, hackDeviceCount)); - } if (deployAsIsInfo != null && deployAsIsInfo.getProperties() != null) { totalChangeDevices++; } @@ -1934,17 +1925,18 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa if (vmSpec.getType() != VirtualMachine.Type.User) { // system VM needs a patch ISO totalChangeDevices++; - } else if (!deployAsIs) { + } else { volIso = getIsoDiskTO(disks); - if (volIso == null) + if (volIso == null && !deployAsIs) { totalChangeDevices++; + } } VirtualMachineConfigSpec vmConfigSpec = new VirtualMachineConfigSpec(); int i = 0; - int ideUnitNumber = 0; - int scsiUnitNumber = 0; + int ideUnitNumber = !deployAsIs ? 0 : vmMo.getNextIDEDeviceNumber(); + int scsiUnitNumber = !deployAsIs ? 0 : vmMo.getNextScsiDiskDeviceNumber(); int ideControllerKey = vmMo.getIDEDeviceControllerKey(); int scsiControllerKey = vmMo.getScsiDeviceControllerKeyNoException(); VirtualDeviceConfigSpec[] deviceConfigSpecArray = new VirtualDeviceConfigSpec[totalChangeDevices]; @@ -1977,214 +1969,214 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } else { vmConfigSpec.setCpuHotAddEnabled(vmMo.isCpuHotAddSupported(guestOsId)); } + } - configNestedHVSupport(vmMo, vmSpec, vmConfigSpec); + configNestedHVSupport(vmMo, vmSpec, vmConfigSpec); - int controllerKey; + // + // Setup ISO device + // - // - // Setup ISO device - // + // prepare systemvm patch ISO + if (vmSpec.getType() != VirtualMachine.Type.User) { + // attach ISO (for patching of system VM) + Pair secStoreUrlAndId = mgr.getSecondaryStorageStoreUrlAndId(Long.parseLong(_dcId)); + String secStoreUrl = secStoreUrlAndId.first(); + Long secStoreId = secStoreUrlAndId.second(); + if (secStoreUrl == null) { + String msg = "secondary storage for dc " + _dcId + " is not ready yet?"; + throw new Exception(msg); + } + mgr.prepareSecondaryStorageStore(secStoreUrl, secStoreId); - // prepare systemvm patch ISO - if (vmSpec.getType() != VirtualMachine.Type.User) { - // attach ISO (for patching of system VM) - Pair secStoreUrlAndId = mgr.getSecondaryStorageStoreUrlAndId(Long.parseLong(_dcId)); - String secStoreUrl = secStoreUrlAndId.first(); - Long secStoreId = secStoreUrlAndId.second(); - if (secStoreUrl == null) { - String msg = "secondary storage for dc " + _dcId + " is not ready yet?"; - throw new Exception(msg); + ManagedObjectReference morSecDs = prepareSecondaryDatastoreOnHost(secStoreUrl); + if (morSecDs == null) { + String msg = "Failed to prepare secondary storage on host, secondary store url: " + secStoreUrl; + throw new Exception(msg); + } + DatastoreMO secDsMo = new DatastoreMO(hyperHost.getContext(), morSecDs); + + deviceConfigSpecArray[i] = new VirtualDeviceConfigSpec(); + Pair isoInfo = VmwareHelper.prepareIsoDevice(vmMo, + String.format("[%s] systemvm/%s", secDsMo.getName(), mgr.getSystemVMIsoFileNameOnDatastore()), secDsMo.getMor(), true, true, ideUnitNumber++, i + 1); + deviceConfigSpecArray[i].setDevice(isoInfo.first()); + if (isoInfo.second()) { + if (s_logger.isDebugEnabled()) + s_logger.debug("Prepare ISO volume at new device " + _gson.toJson(isoInfo.first())); + deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.ADD); + } else { + if (s_logger.isDebugEnabled()) + s_logger.debug("Prepare ISO volume at existing device " + _gson.toJson(isoInfo.first())); + deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.EDIT); + } + i++; + } else if (!deployAsIs) { + // Note: we will always plug a CDROM device + if (volIso != null) { + for (DiskTO vol : disks) { + if (vol.getType() == Volume.Type.ISO) { + + TemplateObjectTO iso = (TemplateObjectTO) vol.getData(); + + if (iso.getPath() != null && !iso.getPath().isEmpty()) { + DataStoreTO imageStore = iso.getDataStore(); + if (!(imageStore instanceof NfsTO)) { + s_logger.debug("unsupported protocol"); + throw new Exception("unsupported protocol"); + } + NfsTO nfsImageStore = (NfsTO) imageStore; + String isoPath = nfsImageStore.getUrl() + File.separator + iso.getPath(); + Pair isoDatastoreInfo = getIsoDatastoreInfo(hyperHost, isoPath); + assert (isoDatastoreInfo != null); + assert (isoDatastoreInfo.second() != null); + + deviceConfigSpecArray[i] = new VirtualDeviceConfigSpec(); + Pair isoInfo = + VmwareHelper.prepareIsoDevice(vmMo, isoDatastoreInfo.first(), isoDatastoreInfo.second(), true, true, ideUnitNumber++, i + 1); + deviceConfigSpecArray[i].setDevice(isoInfo.first()); + if (isoInfo.second()) { + if (s_logger.isDebugEnabled()) + s_logger.debug("Prepare ISO volume at new device " + _gson.toJson(isoInfo.first())); + deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.ADD); + } else { + if (s_logger.isDebugEnabled()) + s_logger.debug("Prepare ISO volume at existing device " + _gson.toJson(isoInfo.first())); + deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.EDIT); + } + } + i++; + } } - mgr.prepareSecondaryStorageStore(secStoreUrl, secStoreId); - - ManagedObjectReference morSecDs = prepareSecondaryDatastoreOnHost(secStoreUrl); - if (morSecDs == null) { - String msg = "Failed to prepare secondary storage on host, secondary store url: " + secStoreUrl; - throw new Exception(msg); - } - DatastoreMO secDsMo = new DatastoreMO(hyperHost.getContext(), morSecDs); - + } else { deviceConfigSpecArray[i] = new VirtualDeviceConfigSpec(); - Pair isoInfo = VmwareHelper.prepareIsoDevice(vmMo, - String.format("[%s] systemvm/%s", secDsMo.getName(), mgr.getSystemVMIsoFileNameOnDatastore()), secDsMo.getMor(), true, true, ideUnitNumber++, i + 1); + Pair isoInfo = VmwareHelper.prepareIsoDevice(vmMo, null, null, true, true, ideUnitNumber++, i + 1); deviceConfigSpecArray[i].setDevice(isoInfo.first()); if (isoInfo.second()) { if (s_logger.isDebugEnabled()) - s_logger.debug("Prepare ISO volume at new device " + _gson.toJson(isoInfo.first())); + s_logger.debug("Prepare ISO volume at existing device " + _gson.toJson(isoInfo.first())); + deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.ADD); } else { if (s_logger.isDebugEnabled()) s_logger.debug("Prepare ISO volume at existing device " + _gson.toJson(isoInfo.first())); + deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.EDIT); } i++; + } + } + + int controllerKey; + + // + // Setup ROOT/DATA disk devices + // + for (DiskTO vol : sortedDisks) { + if (vol.getType() == Volume.Type.ISO || deployAsIs && vol.getType() == Volume.Type.ROOT) { + continue; + } + + VirtualMachineDiskInfo matchingExistingDisk = getMatchingExistingDisk(diskInfoBuilder, vol, hyperHost, context); + controllerKey = getDiskController(matchingExistingDisk, vol, vmSpec, ideControllerKey, scsiControllerKey); + String diskController = getDiskController(vmMo, matchingExistingDisk, vol, controllerInfo); + + if (DiskControllerType.getType(diskController) == DiskControllerType.osdefault) { + diskController = vmMo.getRecommendedDiskController(null); + } + if (DiskControllerType.getType(diskController) == DiskControllerType.ide) { + controllerKey = vmMo.getIDEControllerKey(ideUnitNumber); + if (vol.getType() == Volume.Type.DATADISK) { + // Could be result of flip due to user configured setting or "osdefault" for data disks + // Ensure maximum of 2 data volumes over IDE controller, 3 includeing root volume + if (vmMo.getNumberOfVirtualDisks() > 3) { + throw new CloudRuntimeException("Found more than 3 virtual disks attached to this VM [" + vmMo.getVmName() + "]. Unable to implement the disks over " + + diskController + " controller, as maximum number of devices supported over IDE controller is 4 includeing CDROM device."); + } + } } else { - // Note: we will always plug a CDROM device - if (volIso != null) { - for (DiskTO vol : disks) { - if (vol.getType() == Volume.Type.ISO) { + if (VmwareHelper.isReservedScsiDeviceNumber(scsiUnitNumber)) { + scsiUnitNumber++; + } - TemplateObjectTO iso = (TemplateObjectTO) vol.getData(); - - if (iso.getPath() != null && !iso.getPath().isEmpty()) { - DataStoreTO imageStore = iso.getDataStore(); - if (!(imageStore instanceof NfsTO)) { - s_logger.debug("unsupported protocol"); - throw new Exception("unsupported protocol"); - } - NfsTO nfsImageStore = (NfsTO) imageStore; - String isoPath = nfsImageStore.getUrl() + File.separator + iso.getPath(); - Pair isoDatastoreInfo = getIsoDatastoreInfo(hyperHost, isoPath); - assert (isoDatastoreInfo != null); - assert (isoDatastoreInfo.second() != null); - - deviceConfigSpecArray[i] = new VirtualDeviceConfigSpec(); - Pair isoInfo = - VmwareHelper.prepareIsoDevice(vmMo, isoDatastoreInfo.first(), isoDatastoreInfo.second(), true, true, ideUnitNumber++, i + 1); - deviceConfigSpecArray[i].setDevice(isoInfo.first()); - if (isoInfo.second()) { - if (s_logger.isDebugEnabled()) - s_logger.debug("Prepare ISO volume at new device " + _gson.toJson(isoInfo.first())); - deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.ADD); - } else { - if (s_logger.isDebugEnabled()) - s_logger.debug("Prepare ISO volume at existing device " + _gson.toJson(isoInfo.first())); - deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.EDIT); - } - } - i++; - } - } - } else { - deviceConfigSpecArray[i] = new VirtualDeviceConfigSpec(); - Pair isoInfo = VmwareHelper.prepareIsoDevice(vmMo, null, null, true, true, ideUnitNumber++, i + 1); - deviceConfigSpecArray[i].setDevice(isoInfo.first()); - if (isoInfo.second()) { - if (s_logger.isDebugEnabled()) - s_logger.debug("Prepare ISO volume at existing device " + _gson.toJson(isoInfo.first())); - - deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.ADD); - } else { - if (s_logger.isDebugEnabled()) - s_logger.debug("Prepare ISO volume at existing device " + _gson.toJson(isoInfo.first())); - - deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.EDIT); - } - i++; + controllerKey = vmMo.getScsiDiskControllerKeyNoException(diskController, scsiUnitNumber); + if (controllerKey == -1) { + // This may happen for ROOT legacy VMs which doesn't have recommended disk controller when global configuration parameter 'vmware.root.disk.controller' is set to "osdefault" + // Retrieve existing controller and use. + Ternary vmScsiControllerInfo = vmMo.getScsiControllerInfo(); + DiskControllerType existingControllerType = vmScsiControllerInfo.third(); + controllerKey = vmMo.getScsiDiskControllerKeyNoException(existingControllerType.toString(), scsiUnitNumber); } } + if (!hasSnapshot) { + deviceConfigSpecArray[i] = new VirtualDeviceConfigSpec(); + VolumeObjectTO volumeTO = (VolumeObjectTO) vol.getData(); + DataStoreTO primaryStore = volumeTO.getDataStore(); + Map details = vol.getDetails(); + boolean managed = false; + String iScsiName = null; - // - // Setup ROOT/DATA disk devices - // - for (DiskTO vol : sortedDisks) { - if (vol.getType() == Volume.Type.ISO) - continue; - - VirtualMachineDiskInfo matchingExistingDisk = getMatchingExistingDisk(diskInfoBuilder, vol, hyperHost, context); - controllerKey = getDiskController(matchingExistingDisk, vol, vmSpec, ideControllerKey, scsiControllerKey); - String diskController = getDiskController(vmMo, matchingExistingDisk, vol, controllerInfo); - - if (DiskControllerType.getType(diskController) == DiskControllerType.osdefault) { - diskController = vmMo.getRecommendedDiskController(null); + if (details != null) { + managed = Boolean.parseBoolean(details.get(DiskTO.MANAGED)); + iScsiName = details.get(DiskTO.IQN); } - if (DiskControllerType.getType(diskController) == DiskControllerType.ide) { - controllerKey = vmMo.getIDEControllerKey(ideUnitNumber); - if (vol.getType() == Volume.Type.DATADISK) { - // Could be result of flip due to user configured setting or "osdefault" for data disks - // Ensure maximum of 2 data volumes over IDE controller, 3 includeing root volume - if (vmMo.getNumberOfVirtualDisks() > 3) { - throw new CloudRuntimeException("Found more than 3 virtual disks attached to this VM [" + vmMo.getVmName() + "]. Unable to implement the disks over " - + diskController + " controller, as maximum number of devices supported over IDE controller is 4 includeing CDROM device."); - } - } + + // if the storage is managed, iScsiName should not be null + String datastoreName = managed ? VmwareResource.getDatastoreName(iScsiName) : primaryStore.getUuid(); + Pair volumeDsDetails = dataStoresDetails.get(datastoreName); + + assert (volumeDsDetails != null); + + String[] diskChain = syncDiskChain(dcMo, vmMo, vmSpec, vol, matchingExistingDisk, dataStoresDetails); + + int deviceNumber = -1; + if (controllerKey == vmMo.getIDEControllerKey(ideUnitNumber)) { + deviceNumber = ideUnitNumber % VmwareHelper.MAX_ALLOWED_DEVICES_IDE_CONTROLLER; + ideUnitNumber++; } else { - if (VmwareHelper.isReservedScsiDeviceNumber(scsiUnitNumber)) { - scsiUnitNumber++; - } - - controllerKey = vmMo.getScsiDiskControllerKeyNoException(diskController, scsiUnitNumber); - if (controllerKey == -1) { - // This may happen for ROOT legacy VMs which doesn't have recommended disk controller when global configuration parameter 'vmware.root.disk.controller' is set to "osdefault" - // Retrieve existing controller and use. - Ternary vmScsiControllerInfo = vmMo.getScsiControllerInfo(); - DiskControllerType existingControllerType = vmScsiControllerInfo.third(); - controllerKey = vmMo.getScsiDiskControllerKeyNoException(existingControllerType.toString(), scsiUnitNumber); - } + deviceNumber = scsiUnitNumber % VmwareHelper.MAX_ALLOWED_DEVICES_SCSI_CONTROLLER; + scsiUnitNumber++; } - if (!hasSnapshot) { - deviceConfigSpecArray[i] = new VirtualDeviceConfigSpec(); - VolumeObjectTO volumeTO = (VolumeObjectTO) vol.getData(); - DataStoreTO primaryStore = volumeTO.getDataStore(); - Map details = vol.getDetails(); - boolean managed = false; - String iScsiName = null; + VirtualDevice device = VmwareHelper.prepareDiskDevice(vmMo, null, controllerKey, diskChain, volumeDsDetails.first(), deviceNumber, i + 1); - if (details != null) { - managed = Boolean.parseBoolean(details.get(DiskTO.MANAGED)); - iScsiName = details.get(DiskTO.IQN); - } + if (vol.getType() == Volume.Type.ROOT) + rootDiskTO = vol; + deviceConfigSpecArray[i].setDevice(device); + deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.ADD); - // if the storage is managed, iScsiName should not be null - String datastoreName = managed ? VmwareResource.getDatastoreName(iScsiName) : primaryStore.getUuid(); - Pair volumeDsDetails = dataStoresDetails.get(datastoreName); + if (s_logger.isDebugEnabled()) + s_logger.debug("Prepare volume at new device " + _gson.toJson(device)); - assert (volumeDsDetails != null); - - String[] diskChain = syncDiskChain(dcMo, vmMo, vmSpec, vol, matchingExistingDisk, dataStoresDetails); - - int deviceNumber = -1; - if (controllerKey == vmMo.getIDEControllerKey(ideUnitNumber)) { - deviceNumber = ideUnitNumber % VmwareHelper.MAX_ALLOWED_DEVICES_IDE_CONTROLLER; - ideUnitNumber++; - } else { - deviceNumber = scsiUnitNumber % VmwareHelper.MAX_ALLOWED_DEVICES_SCSI_CONTROLLER; - scsiUnitNumber++; - } - - VirtualDevice device = VmwareHelper.prepareDiskDevice(vmMo, null, controllerKey, diskChain, volumeDsDetails.first(), deviceNumber, i + 1); - - if (vol.getType() == Volume.Type.ROOT) - rootDiskTO = vol; - deviceConfigSpecArray[i].setDevice(device); - deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.ADD); - - if (s_logger.isDebugEnabled()) - s_logger.debug("Prepare volume at new device " + _gson.toJson(device)); - - i++; - } else { - if (controllerKey == vmMo.getIDEControllerKey(ideUnitNumber)) - ideUnitNumber++; - else - scsiUnitNumber++; - } + i++; + } else { + if (controllerKey == vmMo.getIDEControllerKey(ideUnitNumber)) + ideUnitNumber++; + else + scsiUnitNumber++; } + } - // - // Setup USB devices - // - if (guestOsId.startsWith("darwin")) { //Mac OS - VirtualDevice[] devices = vmMo.getMatchedDevices(new Class[]{VirtualUSBController.class}); - if (devices.length == 0) { - s_logger.debug("No USB Controller device on VM Start. Add USB Controller device for Mac OS VM " + vmInternalCSName); + // + // Setup USB devices + // + if (StringUtils.isNotBlank(guestOsId) && guestOsId.startsWith("darwin")) { //Mac OS + VirtualDevice[] devices = vmMo.getMatchedDevices(new Class[]{VirtualUSBController.class}); + if (devices.length == 0) { + s_logger.debug("No USB Controller device on VM Start. Add USB Controller device for Mac OS VM " + vmInternalCSName); - //For Mac OS X systems, the EHCI+UHCI controller is enabled by default and is required for USB mouse and keyboard access. - VirtualDevice usbControllerDevice = VmwareHelper.prepareUSBControllerDevice(); - deviceConfigSpecArray[i] = new VirtualDeviceConfigSpec(); - deviceConfigSpecArray[i].setDevice(usbControllerDevice); - deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.ADD); + //For Mac OS X systems, the EHCI+UHCI controller is enabled by default and is required for USB mouse and keyboard access. + VirtualDevice usbControllerDevice = VmwareHelper.prepareUSBControllerDevice(); + deviceConfigSpecArray[i] = new VirtualDeviceConfigSpec(); + deviceConfigSpecArray[i].setDevice(usbControllerDevice); + deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.ADD); - if (s_logger.isDebugEnabled()) - s_logger.debug("Prepare USB controller at new device " + _gson.toJson(deviceConfigSpecArray[i])); + if (s_logger.isDebugEnabled()) + s_logger.debug("Prepare USB controller at new device " + _gson.toJson(deviceConfigSpecArray[i])); - i++; - } else { - s_logger.debug("USB Controller device exists on VM Start for Mac OS VM " + vmInternalCSName); - } + i++; + } else { + s_logger.debug("USB Controller device exists on VM Start for Mac OS VM " + vmInternalCSName); } } @@ -2410,6 +2402,21 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } } + private int getDisksChangesNumberFromDisksSpec(DiskTO[] disks, boolean deployAsIs) { + if (!deployAsIs) { + return disks.length; + } else { + int datadisksNumber = 0; + if (ArrayUtils.isNotEmpty(disks)) { + List datadisks = Arrays.stream(disks).filter(x -> x.getType() == Volume.Type.DATADISK).collect(Collectors.toList()); + if (CollectionUtils.isNotEmpty(datadisks)) { + datadisksNumber = datadisks.size(); + } + } + return datadisksNumber; + } + } + /** * Configure VNC */ @@ -2452,7 +2459,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa return guestOsId; } - private Pair getControllerInfoFromVmSpec(VirtualMachineTO vmSpec, boolean deployAsIs) throws CloudRuntimeException { + private Pair getControllerInfoFromVmSpec(VirtualMachineTO vmSpec) throws CloudRuntimeException { String dataDiskController = vmSpec.getDetails().get(VmDetailConstants.DATA_DISK_CONTROLLER); String rootDiskController = vmSpec.getDetails().get(VmDetailConstants.ROOT_DISK_CONTROLLER); @@ -2473,7 +2480,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa throw new CloudRuntimeException("Invalid data disk controller detected : " + dataDiskController); } - return new Pair(rootDiskController, dataDiskController); + return new Pair<>(rootDiskController, dataDiskController); } private String getBootModeFromVmSpec(VirtualMachineTO vmSpec, boolean deployAsIs) { diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java index fab928c0520..2013c75b163 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -810,8 +810,8 @@ public class VmwareStorageProcessor implements StorageProcessor { String vmdkName = volume.getName(); String vmdkFileBaseName = null; - if (template.isDeployAsIs()) { - s_logger.info("Volume from deploy-as-is template, no need to create the volume at this point, VM will be cloned"); + if (template.isDeployAsIs() && volume.getVolumeType() == Volume.Type.ROOT) { + s_logger.info("ROOT Volume from deploy-as-is template, no need to create the volume at this point, will be cloned from template"); } else { if (srcStore == null) { // create a root volume for blank VM (created from ISO) From 89aa25d023608cc84aefb9dd229cb101f7e042a0 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Thu, 24 Sep 2020 16:05:06 +0530 Subject: [PATCH 101/164] VM deployment from ISO not working on VSAN65, fixed by adding a backup to create disk using old way --- .../java/com/cloud/storage/resource/VmwareStorageProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java index 2013c75b163..5fc71d379d8 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -2348,7 +2348,7 @@ public class VmwareStorageProcessor implements StorageProcessor { VirtualMachineMO vmMo = null; String volumeUuid = UUID.randomUUID().toString().replace("-", ""); - String volumeDatastorePath = dsMo.getDatastorePath(volumeUuid + ".vmdk"); + String volumeDatastorePath = VmwareStorageLayoutHelper.getDatastorePathBaseFolderFromVmdkFileName(dsMo, volumeUuid + ".vmdk"); VolumeObjectTO newVol = new VolumeObjectTO(); try { From 41354227e27c8b2bf3aaa3f05deaa7e18446620c Mon Sep 17 00:00:00 2001 From: nvazquez Date: Fri, 25 Sep 2020 01:41:47 -0300 Subject: [PATCH 102/164] Handle guest OS read from deploy-as-is OVF descriptor --- .../cloud/agent/api/storage/OVFHelper.java | 49 ++- .../cloud/agent/api/to/OVFInformationTO.java | 95 ++++ .../OVFVirtualHardwareSectionTO.java | 9 +- .../cloud/deployasis/DeployAsIsConstants.java | 1 + .../user/template/RegisterTemplateCmd.java | 4 + .../agent/api/storage/OVFHelperTest.java | 415 +++++++++--------- .../agent/api/storage/DownloadAnswer.java | 57 +-- .../cloud/storage/template/OVAProcessor.java | 41 +- .../com/cloud/storage/template/Processor.java | 13 +- .../agent/api/storage/DownloadAnswerTest.java | 7 +- .../cloud/storage/dao/GuestOSCategoryDao.java | 1 + .../storage/dao/GuestOSCategoryDaoImpl.java | 9 +- .../storage/dao/GuestOSHypervisorDao.java | 7 + .../storage/dao/GuestOSHypervisorDaoImpl.java | 37 ++ .../META-INF/db/schema-41400to41500.sql | 5 +- .../deployasis/DeployAsIsHelperImpl.java | 212 ++++++++- .../cloud/template/TemplateAdapterBase.java | 8 + .../storage/template/DownloadManagerImpl.java | 81 +--- 18 files changed, 684 insertions(+), 367 deletions(-) create mode 100644 api/src/main/java/com/cloud/agent/api/to/OVFInformationTO.java diff --git a/api/src/main/java/com/cloud/agent/api/storage/OVFHelper.java b/api/src/main/java/com/cloud/agent/api/storage/OVFHelper.java index 45eff89b0c9..8d4f780efed 100644 --- a/api/src/main/java/com/cloud/agent/api/storage/OVFHelper.java +++ b/api/src/main/java/com/cloud/agent/api/storage/OVFHelper.java @@ -46,6 +46,7 @@ import com.cloud.agent.api.to.deployasis.OVFVirtualHardwareItemTO; import com.cloud.agent.api.to.deployasis.OVFVirtualHardwareSectionTO; import com.cloud.configuration.Resource.ResourceType; import com.cloud.exception.InternalErrorException; +import com.cloud.utils.Pair; import com.cloud.utils.compression.CompressionUtil; import com.cloud.agent.api.to.deployasis.OVFNetworkTO; import org.apache.commons.collections.CollectionUtils; @@ -217,6 +218,12 @@ public class OVFHelper { return getConfigurableOVFPropertiesFromDocument(doc); } + protected Pair getOperatingSystemInfoFromXmlString(final String ovfString) throws ParserConfigurationException, IOException, SAXException { + InputSource is = new InputSource(new StringReader(ovfString)); + final Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(is); + return getOperatingSystemInfoFromDocument(doc); + } + protected List getOVFDeploymentOptionsFromXmlString(final String ovfString) throws ParserConfigurationException, IOException, SAXException { InputSource is = new InputSource(new StringReader(ovfString)); final Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(is); @@ -229,6 +236,12 @@ public class OVFHelper { return getVirtualHardwareItemsFromDocumentTree(doc); } + protected OVFVirtualHardwareSectionTO getVirtualHardwareSectionFromXmlString(final String ovfString) throws ParserConfigurationException, IOException, SAXException { + InputSource is = new InputSource(new StringReader(ovfString)); + final Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(is); + return getVirtualHardwareSectionFromDocument(doc); + } + protected List getOVFEulaSectionFromXmlString(final String ovfString) throws ParserConfigurationException, IOException, SAXException { InputSource is = new InputSource(new StringReader(ovfString)); final Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(is); @@ -726,7 +739,26 @@ public class OVFHelper { if (CollectionUtils.isNotEmpty(items)) { commonItems = items.stream().filter(x -> StringUtils.isBlank(x.getConfigurationIds())).collect(Collectors.toList()); } - return new OVFVirtualHardwareSectionTO(configurations, commonItems); + String minimumHardwareVersion = getMinimumHardwareVersionFromDocumentTree(doc); + return new OVFVirtualHardwareSectionTO(configurations, commonItems, minimumHardwareVersion); + } + + private String getMinimumHardwareVersionFromDocumentTree(Document doc) { + String version = null; + if (doc != null) { + NodeList systemNodeList = doc.getElementsByTagName("System"); + if (systemNodeList.getLength() != 0) { + Node systemItem = systemNodeList.item(0); + String hardwareVersions = getChildNodeValue(systemItem, "VirtualSystemType"); + if (StringUtils.isNotBlank(hardwareVersions)) { + String[] versions = hardwareVersions.split(","); + // Order the hardware versions and retrieve the minimum version + List versionsList = Arrays.stream(versions).sorted().collect(Collectors.toList()); + version = versionsList.get(0); + } + } + } + return version; } private List getDeploymentOptionsFromDocumentTree(Document doc) { @@ -869,6 +901,21 @@ public class OVFHelper { return eulas; } + public Pair getOperatingSystemInfoFromDocument(Document doc) { + if (doc == null) { + return null; + } + NodeList guesOsList = doc.getElementsByTagName("OperatingSystemSection"); + if (guesOsList.getLength() == 0) { + return null; + } + Node guestOsNode = guesOsList.item(0); + Element guestOsElement = (Element) guestOsNode; + String osType = getNodeAttribute(guestOsElement, "vmw", "osType"); + String description = getChildNodeValue(guestOsNode, "Description"); + return new Pair<>(osType, description); + } + class OVFFile { // public String _href; diff --git a/api/src/main/java/com/cloud/agent/api/to/OVFInformationTO.java b/api/src/main/java/com/cloud/agent/api/to/OVFInformationTO.java new file mode 100644 index 00000000000..412d362441e --- /dev/null +++ b/api/src/main/java/com/cloud/agent/api/to/OVFInformationTO.java @@ -0,0 +1,95 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package com.cloud.agent.api.to; + +import com.cloud.agent.api.LogLevel; +import com.cloud.agent.api.to.deployasis.OVFEulaSectionTO; +import com.cloud.agent.api.to.deployasis.OVFNetworkTO; +import com.cloud.agent.api.to.deployasis.OVFPropertyTO; +import com.cloud.agent.api.to.deployasis.OVFVirtualHardwareSectionTO; +import com.cloud.utils.Pair; + +import java.util.List; + +public class OVFInformationTO { + + @LogLevel(LogLevel.Log4jLevel.Off) + private List properties; + @LogLevel(LogLevel.Log4jLevel.Off) + private List networks; + @LogLevel(LogLevel.Log4jLevel.Off) + private List disks; + @LogLevel(LogLevel.Log4jLevel.Off) + private OVFVirtualHardwareSectionTO hardwareSection; + @LogLevel(LogLevel.Log4jLevel.Off) + private List eulaSections; + @LogLevel(LogLevel.Log4jLevel.Off) + private Pair guestOsInfo; + + public OVFInformationTO() { + } + + public List getProperties() { + return properties; + } + + public void setProperties(List properties) { + this.properties = properties; + } + + public List getNetworks() { + return networks; + } + + public void setNetworks(List networks) { + this.networks = networks; + } + + public List getDisks() { + return disks; + } + + public void setDisks(List disks) { + this.disks = disks; + } + + public OVFVirtualHardwareSectionTO getHardwareSection() { + return hardwareSection; + } + + public void setHardwareSection(OVFVirtualHardwareSectionTO hardwareSection) { + this.hardwareSection = hardwareSection; + } + + public List getEulaSections() { + return eulaSections; + } + + public void setEulaSections(List eulaSections) { + this.eulaSections = eulaSections; + } + + public Pair getGuestOsInfo() { + return guestOsInfo; + } + + public void setGuestOsInfo(Pair guestOsInfo) { + this.guestOsInfo = guestOsInfo; + } +} diff --git a/api/src/main/java/com/cloud/agent/api/to/deployasis/OVFVirtualHardwareSectionTO.java b/api/src/main/java/com/cloud/agent/api/to/deployasis/OVFVirtualHardwareSectionTO.java index 2698f8c5e18..4cdbf68ed9c 100644 --- a/api/src/main/java/com/cloud/agent/api/to/deployasis/OVFVirtualHardwareSectionTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/deployasis/OVFVirtualHardwareSectionTO.java @@ -25,12 +25,15 @@ public class OVFVirtualHardwareSectionTO implements TemplateDeployAsIsInformatio public OVFVirtualHardwareSectionTO() { } + private String minimiumHardwareVersion; private List configurations; private List commonHardwareItems; - public OVFVirtualHardwareSectionTO(List configurations, List commonHardwareItems) { + public OVFVirtualHardwareSectionTO(List configurations, List commonHardwareItems, + String minimumHardwareVersion) { this.configurations = configurations; this.commonHardwareItems = commonHardwareItems; + this.minimiumHardwareVersion = minimumHardwareVersion; } public List getConfigurations() { @@ -40,4 +43,8 @@ public class OVFVirtualHardwareSectionTO implements TemplateDeployAsIsInformatio public List getCommonHardwareItems() { return commonHardwareItems; } + + public String getMinimiumHardwareVersion() { + return minimiumHardwareVersion; + } } \ No newline at end of file diff --git a/api/src/main/java/com/cloud/deployasis/DeployAsIsConstants.java b/api/src/main/java/com/cloud/deployasis/DeployAsIsConstants.java index 3dc961b9de8..23d286f5a7b 100644 --- a/api/src/main/java/com/cloud/deployasis/DeployAsIsConstants.java +++ b/api/src/main/java/com/cloud/deployasis/DeployAsIsConstants.java @@ -24,4 +24,5 @@ public interface DeployAsIsConstants { String HARDWARE_ITEM_PREFIX = "hardware-item-"; String EULA_PREFIX = "eula-"; + String DEFAULT_GUEST_OS_DEPLOY_AS_IS = "OVF Configured OS"; } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java index ca08c2a5ded..6decb945496 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java @@ -347,5 +347,9 @@ public class RegisterTemplateCmd extends BaseCmd implements UserCmd { throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Parameter directdownload is only allowed for KVM templates"); } + + if (isDeployAsIs() && !getHypervisor().equalsIgnoreCase(Hypervisor.HypervisorType.VMware.toString())) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Parameter deployasis is only allowed for VMware templates"); + } } } diff --git a/api/src/test/java/com/cloud/agent/api/storage/OVFHelperTest.java b/api/src/test/java/com/cloud/agent/api/storage/OVFHelperTest.java index e28680784d7..5e7b7cf9bc9 100644 --- a/api/src/test/java/com/cloud/agent/api/storage/OVFHelperTest.java +++ b/api/src/test/java/com/cloud/agent/api/storage/OVFHelperTest.java @@ -20,6 +20,8 @@ import com.cloud.agent.api.to.deployasis.OVFConfigurationTO; import com.cloud.agent.api.to.deployasis.OVFEulaSectionTO; import com.cloud.agent.api.to.deployasis.OVFPropertyTO; import com.cloud.agent.api.to.deployasis.OVFVirtualHardwareItemTO; +import com.cloud.agent.api.to.deployasis.OVFVirtualHardwareSectionTO; +import com.cloud.utils.Pair; import org.junit.Assert; import org.junit.Test; import org.xml.sax.SAXException; @@ -62,204 +64,210 @@ public class OVFHelperTest { " "; private String ovfFileVirtualHardwareSection = + "\n" + + "\n" + + " The kind of installed guest operating system\n" + + " Other 2.6x Linux (64-bit)\n" + + "\n" + "\n" + - " Virtual hardware requirements\n" + - " \n" + - " Virtual Hardware Family\n" + - " 0\n" + - " ASAv\n" + - " vmx-08,vmx-09\n" + - " \n" + - " \n" + - " hertz * 10^6\n" + - " Number of Virtual CPUs\n" + - " 1 virtual CPU(s)\n" + - " 1\n" + - " 5000\n" + - " 1000\n" + - " 3\n" + - " 1\n" + - " \n" + - " \n" + - " hertz * 10^6\n" + - " Number of Virtual CPUs\n" + - " 4 virtual CPU(s)\n" + - " 1\n" + - " 20000\n" + - " 1000\n" + - " 3\n" + - " 4\n" + - " \n" + - " \n" + - " byte * 2^20\n" + - " Memory Size\n" + - " 2048MB of memory\n" + - " 2\n" + - " 2048\n" + - " 2048\n" + - " 4\n" + - " 2048\n" + - " \n" + - " \n" + - " byte * 2^20\n" + - " Memory Size\n" + - " 8192MB of memory\n" + - " 2\n" + - " 8192\n" + - " 8192\n" + - " 4\n" + - " 8192\n" + - " \n" + - " \n" + - " 0\n" + - " SCSI Controller\n" + - " SCSI controller 0\n" + - " 3\n" + - " lsilogic\n" + - " 6\n" + - " \n" + - " \n" + - " 0\n" + - " IDE Controller\n" + - " IDE 0\n" + - " 4\n" + - " 5\n" + - " \n" + - " \n" + - " 0\n" + - " true\n" + - " CD/DVD Drive\n" + - " 5\n" + - " 4\n" + - " 15\n" + - " \n" + - " \n" + - " 1\n" + - " true\n" + - " CD/DVD Drive\n" + - " ovf:/file/file3\n" + - " 18\n" + - " 4\n" + - " 15\n" + - " \n" + - " \n" + - " 7\n" + - " true\n" + - " Management0-0\n" + - " E1000 Ethernet adapter on \"Management Network\"\n" + - " Network adapter 1\n" + - " 6\n" + - " E1000\n" + - " 10\n" + - " \n" + - " \n" + - " 0\n" + - " Hard disk 1\n" + - " ovf:/disk/vmdisk1\n" + - " 7\n" + - " 3\n" + - " 17\n" + - " \n" + - " \n" + - " 1\n" + - " Hard disk 2\n" + - " ovf:/disk/vmdisk2\n" + - " 8\n" + - " 3\n" + - " 17\n" + - " \n" + - " \n" + - " 8\n" + - " true\n" + - " GigabitEthernet0-0\n" + - " General purpose E1000 Ethernet adapter\n" + - " Network adapter 2\n" + - " 9\n" + - " E1000\n" + - " 10\n" + - " \n" + - " \n" + - " 9\n" + - " true\n" + - " GigabitEthernet0-1\n" + - " General purpose E1000 Ethernet adapter\n" + - " Network adapter 3\n" + - " 10\n" + - " E1000\n" + - " 10\n" + - " \n" + - " \n" + - " 10\n" + - " true\n" + - " GigabitEthernet0-2\n" + - " General purpose E1000 Ethernet adapter\n" + - " Network adapter 4\n" + - " 11\n" + - " E1000\n" + - " 10\n" + - " \n" + - " \n" + - " 11\n" + - " true\n" + - " GigabitEthernet0-3\n" + - " General purpose E1000 Ethernet adapter\n" + - " Network adapter 5\n" + - " 12\n" + - " E1000\n" + - " 10\n" + - " \n" + - " \n" + - " 12\n" + - " true\n" + - " GigabitEthernet0-4\n" + - " General purpose E1000 Ethernet adapter\n" + - " Network adapter 6\n" + - " 13\n" + - " E1000\n" + - " 10\n" + - " \n" + - " \n" + - " 13\n" + - " true\n" + - " GigabitEthernet0-5\n" + - " General purpose E1000 Ethernet adapter\n" + - " Network adapter 7\n" + - " 14\n" + - " E1000\n" + - " 10\n" + - " \n" + - " \n" + - " 14\n" + - " true\n" + - " GigabitEthernet0-6\n" + - " General purpose E1000 Ethernet adapter\n" + - " Network adapter 8\n" + - " 15\n" + - " E1000\n" + - " 10\n" + - " \n" + - " \n" + - " 15\n" + - " true\n" + - " GigabitEthernet0-7\n" + - " General purpose E1000 Ethernet adapter\n" + - " Network adapter 9\n" + - " 16\n" + - " E1000\n" + - " 10\n" + - " \n" + - " \n" + - " 16\n" + - " true\n" + - " GigabitEthernet0-8\n" + - " Default HA failover E1000 Ethernet adapter, or additional standalone general purpose adapter\n" + - " Network adapter 10\n" + - " 17\n" + - " E1000\n" + - " 10\n" + - " \n" + - " \n" + - " "; + " Virtual hardware requirements\n" + + " \n" + + " Virtual Hardware Family\n" + + " 0\n" + + " ASAv\n" + + " vmx-08,vmx-09\n" + + " \n" + + " \n" + + " hertz * 10^6\n" + + " Number of Virtual CPUs\n" + + " 1 virtual CPU(s)\n" + + " 1\n" + + " 5000\n" + + " 1000\n" + + " 3\n" + + " 1\n" + + " \n" + + " \n" + + " hertz * 10^6\n" + + " Number of Virtual CPUs\n" + + " 4 virtual CPU(s)\n" + + " 1\n" + + " 20000\n" + + " 1000\n" + + " 3\n" + + " 4\n" + + " \n" + + " \n" + + " byte * 2^20\n" + + " Memory Size\n" + + " 2048MB of memory\n" + + " 2\n" + + " 2048\n" + + " 2048\n" + + " 4\n" + + " 2048\n" + + " \n" + + " \n" + + " byte * 2^20\n" + + " Memory Size\n" + + " 8192MB of memory\n" + + " 2\n" + + " 8192\n" + + " 8192\n" + + " 4\n" + + " 8192\n" + + " \n" + + " \n" + + " 0\n" + + " SCSI Controller\n" + + " SCSI controller 0\n" + + " 3\n" + + " lsilogic\n" + + " 6\n" + + " \n" + + " \n" + + " 0\n" + + " IDE Controller\n" + + " IDE 0\n" + + " 4\n" + + " 5\n" + + " \n" + + " \n" + + " 0\n" + + " true\n" + + " CD/DVD Drive\n" + + " 5\n" + + " 4\n" + + " 15\n" + + " \n" + + " \n" + + " 1\n" + + " true\n" + + " CD/DVD Drive\n" + + " ovf:/file/file3\n" + + " 18\n" + + " 4\n" + + " 15\n" + + " \n" + + " \n" + + " 7\n" + + " true\n" + + " Management0-0\n" + + " E1000 Ethernet adapter on \"Management Network\"\n" + + " Network adapter 1\n" + + " 6\n" + + " E1000\n" + + " 10\n" + + " \n" + + " \n" + + " 0\n" + + " Hard disk 1\n" + + " ovf:/disk/vmdisk1\n" + + " 7\n" + + " 3\n" + + " 17\n" + + " \n" + + " \n" + + " 1\n" + + " Hard disk 2\n" + + " ovf:/disk/vmdisk2\n" + + " 8\n" + + " 3\n" + + " 17\n" + + " \n" + + " \n" + + " 8\n" + + " true\n" + + " GigabitEthernet0-0\n" + + " General purpose E1000 Ethernet adapter\n" + + " Network adapter 2\n" + + " 9\n" + + " E1000\n" + + " 10\n" + + " \n" + + " \n" + + " 9\n" + + " true\n" + + " GigabitEthernet0-1\n" + + " General purpose E1000 Ethernet adapter\n" + + " Network adapter 3\n" + + " 10\n" + + " E1000\n" + + " 10\n" + + " \n" + + " \n" + + " 10\n" + + " true\n" + + " GigabitEthernet0-2\n" + + " General purpose E1000 Ethernet adapter\n" + + " Network adapter 4\n" + + " 11\n" + + " E1000\n" + + " 10\n" + + " \n" + + " \n" + + " 11\n" + + " true\n" + + " GigabitEthernet0-3\n" + + " General purpose E1000 Ethernet adapter\n" + + " Network adapter 5\n" + + " 12\n" + + " E1000\n" + + " 10\n" + + " \n" + + " \n" + + " 12\n" + + " true\n" + + " GigabitEthernet0-4\n" + + " General purpose E1000 Ethernet adapter\n" + + " Network adapter 6\n" + + " 13\n" + + " E1000\n" + + " 10\n" + + " \n" + + " \n" + + " 13\n" + + " true\n" + + " GigabitEthernet0-5\n" + + " General purpose E1000 Ethernet adapter\n" + + " Network adapter 7\n" + + " 14\n" + + " E1000\n" + + " 10\n" + + " \n" + + " \n" + + " 14\n" + + " true\n" + + " GigabitEthernet0-6\n" + + " General purpose E1000 Ethernet adapter\n" + + " Network adapter 8\n" + + " 15\n" + + " E1000\n" + + " 10\n" + + " \n" + + " \n" + + " 15\n" + + " true\n" + + " GigabitEthernet0-7\n" + + " General purpose E1000 Ethernet adapter\n" + + " Network adapter 9\n" + + " 16\n" + + " E1000\n" + + " 10\n" + + " \n" + + " \n" + + " 16\n" + + " true\n" + + " GigabitEthernet0-8\n" + + " Default HA failover E1000 Ethernet adapter, or additional standalone general purpose adapter\n" + + " Network adapter 10\n" + + " 17\n" + + " E1000\n" + + " 10\n" + + " \n" + + " \n" + + " \n" + + ""; private String eulaSections = "\n" + @@ -729,4 +737,17 @@ public class OVFHelperTest { List props = ovfHelper.getOVFPropertiesFromXmlString(productSectionWithCategories); Assert.assertEquals(18, props.size()); } + + @Test + public void testGetOperatingSystemInfo() throws IOException, SAXException, ParserConfigurationException { + Pair guestOsPair = ovfHelper.getOperatingSystemInfoFromXmlString(ovfFileVirtualHardwareSection); + Assert.assertEquals("other26xLinux64Guest", guestOsPair.first()); + Assert.assertEquals("Other 2.6x Linux (64-bit)", guestOsPair.second()); + } + + @Test + public void testGetMinimumHardwareVersion() throws IOException, SAXException, ParserConfigurationException { + OVFVirtualHardwareSectionTO hardwareSection = ovfHelper.getVirtualHardwareSectionFromXmlString(ovfFileVirtualHardwareSection); + Assert.assertEquals("vmx-08", hardwareSection.getMinimiumHardwareVersion()); + } } diff --git a/core/src/main/java/com/cloud/agent/api/storage/DownloadAnswer.java b/core/src/main/java/com/cloud/agent/api/storage/DownloadAnswer.java index 9ae1d6dcca5..0c6373134b1 100644 --- a/core/src/main/java/com/cloud/agent/api/storage/DownloadAnswer.java +++ b/core/src/main/java/com/cloud/agent/api/storage/DownloadAnswer.java @@ -20,18 +20,13 @@ package com.cloud.agent.api.storage; import java.io.File; -import java.util.List; import com.cloud.agent.api.Answer; import com.cloud.agent.api.Command; import com.cloud.agent.api.LogLevel; -import com.cloud.agent.api.to.DatadiskTO; -import com.cloud.agent.api.to.deployasis.OVFEulaSectionTO; -import com.cloud.agent.api.to.deployasis.OVFPropertyTO; -import com.cloud.agent.api.to.deployasis.OVFVirtualHardwareSectionTO; +import com.cloud.agent.api.to.OVFInformationTO; import com.cloud.storage.VMTemplateStorageResourceAssoc; import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; -import com.cloud.agent.api.to.deployasis.OVFNetworkTO; public class DownloadAnswer extends Answer { private String jobId; @@ -45,15 +40,7 @@ public class DownloadAnswer extends Answer { private String checkSum; @LogLevel(LogLevel.Log4jLevel.Off) - private List ovfProperties; - @LogLevel(LogLevel.Log4jLevel.Off) - private List networkRequirements; - @LogLevel(LogLevel.Log4jLevel.Off) - private List disks; - @LogLevel(LogLevel.Log4jLevel.Off) - private OVFVirtualHardwareSectionTO ovfHardwareSection; - @LogLevel(LogLevel.Log4jLevel.Off) - private List eulaSections; + private OVFInformationTO ovfInformationTO; public String getCheckSum() { return checkSum; @@ -164,43 +151,11 @@ public class DownloadAnswer extends Answer { return templatePhySicalSize; } - public List getOvfProperties() { - return ovfProperties; + public OVFInformationTO getOvfInformationTO() { + return ovfInformationTO; } - public void setOvfProperties(List ovfProperties) { - this.ovfProperties = ovfProperties; - } - - public List getNetworkRequirements() { - return networkRequirements; - } - - public void setNetworkRequirements(List networkRequirements) { - this.networkRequirements = networkRequirements; - } - - public List getDisks() { - return disks; - } - - public void setDisks(List disks) { - this.disks = disks; - } - - public OVFVirtualHardwareSectionTO getOvfHardwareSection() { - return ovfHardwareSection; - } - - public void setOvfHardwareSection(OVFVirtualHardwareSectionTO ovfHardwareSection) { - this.ovfHardwareSection = ovfHardwareSection; - } - - public List getEulaSections() { - return eulaSections; - } - - public void setEulaSections(List eulaSections) { - this.eulaSections = eulaSections; + public void setOvfInformationTO(OVFInformationTO ovfInformationTO) { + this.ovfInformationTO = ovfInformationTO; } } diff --git a/core/src/main/java/com/cloud/storage/template/OVAProcessor.java b/core/src/main/java/com/cloud/storage/template/OVAProcessor.java index e980293b750..8315ff4ff91 100644 --- a/core/src/main/java/com/cloud/storage/template/OVAProcessor.java +++ b/core/src/main/java/com/cloud/storage/template/OVAProcessor.java @@ -28,6 +28,7 @@ import javax.naming.ConfigurationException; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; +import com.cloud.agent.api.to.OVFInformationTO; import com.cloud.agent.api.to.deployasis.OVFConfigurationTO; import com.cloud.agent.api.to.deployasis.OVFEulaSectionTO; import com.cloud.agent.api.to.deployasis.OVFPropertyTO; @@ -35,6 +36,7 @@ import com.cloud.agent.api.to.deployasis.OVFVirtualHardwareItemTO; import com.cloud.agent.api.to.deployasis.OVFVirtualHardwareSectionTO; import com.cloud.agent.api.to.deployasis.OVFNetworkTO; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import org.w3c.dom.Document; import org.w3c.dom.Element; @@ -107,30 +109,34 @@ public class OVAProcessor extends AdapterBase implements Processor { OVFHelper ovfHelper = new OVFHelper(); Document doc = ovfHelper.getDocumentFromFile(ovfFilePath); - List disks = ovfHelper.getOVFVolumeInfoFromFile(ovfFilePath, doc, null); - if (LOGGER.isTraceEnabled()) { - LOGGER.trace(String.format("Found %d disks in template %s", CollectionUtils.isNotEmpty(disks) ? disks.size() : 0, ovfFilePath)); - } - if (CollectionUtils.isNotEmpty(disks)) { - info.disks = disks; - } + OVFInformationTO ovfInformationTO = createOvfInformationTO(ovfHelper, doc, ovfFilePath); + info.ovfInformationTO = ovfInformationTO; + } + private OVFInformationTO createOvfInformationTO(OVFHelper ovfHelper, Document doc, String ovfFilePath) throws InternalErrorException { + OVFInformationTO ovfInformationTO = new OVFInformationTO(); + + List disks = ovfHelper.getOVFVolumeInfoFromFile(ovfFilePath, doc, null); + if (CollectionUtils.isNotEmpty(disks)) { + if (LOGGER.isTraceEnabled()) { + LOGGER.trace(String.format("Found %d disks in template %s", disks.size(), ovfFilePath)); + } + ovfInformationTO.setDisks(disks); + } List nets = ovfHelper.getNetPrerequisitesFromDocument(doc); if (CollectionUtils.isNotEmpty(nets)) { LOGGER.info("Found " + nets.size() + " prerequisite networks"); - info.networks = nets; + ovfInformationTO.setNetworks(nets); } else if (LOGGER.isTraceEnabled()) { LOGGER.trace(String.format("no net prerequisites found in template %s", ovfFilePath)); } - List ovfProperties = ovfHelper.getConfigurableOVFPropertiesFromDocument(doc); if (CollectionUtils.isNotEmpty(ovfProperties)) { LOGGER.info("Found " + ovfProperties.size() + " configurable OVF properties"); - info.ovfProperties = ovfProperties; + ovfInformationTO.setProperties(ovfProperties); } else if (LOGGER.isTraceEnabled()) { LOGGER.trace(String.format("no ovf properties found in template %s", ovfFilePath)); } - OVFVirtualHardwareSectionTO hardwareSection = ovfHelper.getVirtualHardwareSectionFromDocument(doc); List configurations = hardwareSection.getConfigurations(); if (CollectionUtils.isNotEmpty(configurations)) { @@ -140,12 +146,21 @@ public class OVAProcessor extends AdapterBase implements Processor { if (CollectionUtils.isNotEmpty(hardwareItems)) { LOGGER.info("Found " + hardwareItems.size() + " virtual hardware items"); } - info.hardwareSection = hardwareSection; + if (StringUtils.isNotBlank(hardwareSection.getMinimiumHardwareVersion())) { + LOGGER.info("Found minimum hardware version " + hardwareSection.getMinimiumHardwareVersion()); + } + ovfInformationTO.setHardwareSection(hardwareSection); List eulaSections = ovfHelper.getEulaSectionsFromDocument(doc); if (CollectionUtils.isNotEmpty(eulaSections)) { LOGGER.info("Found " + eulaSections.size() + " license agreements"); - info.eulaSections = eulaSections; + ovfInformationTO.setEulaSections(eulaSections); } + Pair guestOsPair = ovfHelper.getOperatingSystemInfoFromDocument(doc); + if (guestOsPair != null) { + LOGGER.info("Found guest OS information: " + guestOsPair.first() + " - " + guestOsPair.second()); + ovfInformationTO.setGuestOsInfo(guestOsPair); + } + return ovfInformationTO; } private void setFileSystemAccessRights(String templatePath) { diff --git a/core/src/main/java/com/cloud/storage/template/Processor.java b/core/src/main/java/com/cloud/storage/template/Processor.java index 15a3dec0c6e..d8c0dfb94ec 100644 --- a/core/src/main/java/com/cloud/storage/template/Processor.java +++ b/core/src/main/java/com/cloud/storage/template/Processor.java @@ -21,16 +21,11 @@ package com.cloud.storage.template; import java.io.File; import java.io.IOException; -import java.util.List; -import com.cloud.agent.api.to.deployasis.OVFEulaSectionTO; -import com.cloud.agent.api.to.deployasis.OVFPropertyTO; -import com.cloud.agent.api.to.deployasis.OVFVirtualHardwareSectionTO; -import com.cloud.agent.api.to.DatadiskTO; +import com.cloud.agent.api.to.OVFInformationTO; import com.cloud.exception.InternalErrorException; import com.cloud.storage.Storage.ImageFormat; import com.cloud.utils.component.Adapter; -import com.cloud.agent.api.to.deployasis.OVFNetworkTO; /** * Generic interface to process different types of image formats @@ -58,11 +53,7 @@ public interface Processor extends Adapter { public long virtualSize; public String filename; public boolean isCorrupted; - public List ovfProperties; - public List networks; - public List disks; - public OVFVirtualHardwareSectionTO hardwareSection; - public List eulaSections; + public OVFInformationTO ovfInformationTO; } long getVirtualSize(File file) throws IOException; diff --git a/core/src/test/java/com/cloud/agent/api/storage/DownloadAnswerTest.java b/core/src/test/java/com/cloud/agent/api/storage/DownloadAnswerTest.java index c7dcc22572a..ded45f819f3 100644 --- a/core/src/test/java/com/cloud/agent/api/storage/DownloadAnswerTest.java +++ b/core/src/test/java/com/cloud/agent/api/storage/DownloadAnswerTest.java @@ -17,6 +17,7 @@ package com.cloud.agent.api.storage; import com.cloud.agent.api.Answer; +import com.cloud.agent.api.to.OVFInformationTO; import com.cloud.agent.api.to.deployasis.OVFPropertyTO; import com.cloud.serializer.GsonHelper; import com.cloud.storage.VMTemplateStorageResourceAssoc; @@ -49,8 +50,10 @@ public class DownloadAnswerTest { List networks = new ArrayList<>(); networks.add(new OVFNetworkTO()); - answer.setOvfProperties(properties); - answer.setNetworkRequirements(networks); + OVFInformationTO informationTO = new OVFInformationTO(); + informationTO.setProperties(properties); + informationTO.setNetworks(networks); + answer.setOvfInformationTO(informationTO); String json = gson.toJson(answer); Answer received = gson.fromJson(json, Answer.class); diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSCategoryDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSCategoryDao.java index 61fec36d526..d98ef23857b 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSCategoryDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSCategoryDao.java @@ -21,4 +21,5 @@ import com.cloud.utils.db.GenericDao; public interface GuestOSCategoryDao extends GenericDao { + GuestOSCategoryVO findByCategoryName(String categoryName); } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSCategoryDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSCategoryDaoImpl.java index 8ffb54a9873..6fad6c5c47e 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSCategoryDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSCategoryDaoImpl.java @@ -16,7 +16,8 @@ // under the License. package com.cloud.storage.dao; - +import com.cloud.utils.db.QueryBuilder; +import com.cloud.utils.db.SearchCriteria; import org.springframework.stereotype.Component; import com.cloud.storage.GuestOSCategoryVO; @@ -29,4 +30,10 @@ public class GuestOSCategoryDaoImpl extends GenericDaoBase sc = QueryBuilder.create(GuestOSCategoryVO.class); + sc.and(sc.entity().getName(), SearchCriteria.Op.EQ, categoryName); + return sc.find(); + } } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSHypervisorDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSHypervisorDao.java index cd00703ce13..41b95718a0a 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSHypervisorDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSHypervisorDao.java @@ -20,6 +20,8 @@ import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.storage.GuestOSHypervisorVO; import com.cloud.utils.db.GenericDao; +import java.util.List; + public interface GuestOSHypervisorDao extends GenericDao { HypervisorType findHypervisorTypeByGuestOsId(long guestOsId); @@ -31,4 +33,9 @@ public interface GuestOSHypervisorDao extends GenericDao listByOsNameAndHypervisorMinimumVersion(String guestOsName, String hypervisorType, + String minHypervisorVersion); + + List listHypervisorSupportedVersionsFromMinimumVersion(String hypervisorType, String hypervisorVersion); } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSHypervisorDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSHypervisorDaoImpl.java index 699ce0bae02..add4bfc3e7d 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSHypervisorDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSHypervisorDaoImpl.java @@ -16,9 +16,11 @@ // under the License. package com.cloud.storage.dao; +import java.util.ArrayList; import java.util.Date; import java.util.List; +import com.cloud.utils.db.QueryBuilder; import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Component; @@ -36,6 +38,7 @@ public class GuestOSHypervisorDaoImpl extends GenericDaoBase mappingSearch; protected final SearchBuilder userDefinedMappingSearch; protected final SearchBuilder guestOsNameSearch; + protected final SearchBuilder availableHypervisorVersionSearch; protected GuestOSHypervisorDaoImpl() { guestOsSearch = createSearchBuilder(); @@ -60,6 +63,15 @@ public class GuestOSHypervisorDaoImpl extends GenericDaoBase listByOsNameAndHypervisorMinimumVersion(String guestOsName, String hypervisorType, + String minHypervisorVersion) { + final QueryBuilder sc = QueryBuilder.create(GuestOSHypervisorVO.class); + sc.and(sc.entity().getGuestOsName(), SearchCriteria.Op.EQ, guestOsName); + sc.and(sc.entity().getHypervisorType(), SearchCriteria.Op.EQ, hypervisorType); + sc.and(sc.entity().getHypervisorVersion(), SearchCriteria.Op.GTEQ, minHypervisorVersion); + sc.and(sc.entity().getHypervisorVersion(), SearchCriteria.Op.NEQ, "default"); + return sc.list(); + } + + @Override + public List listHypervisorSupportedVersionsFromMinimumVersion(String hypervisorType, String hypervisorVersion) { + List versions = new ArrayList<>(); + SearchCriteria sc = availableHypervisorVersionSearch.create(); + sc.setParameters("hypervisor_type", hypervisorType); + sc.setParameters("hypervisor_version", hypervisorVersion); + Filter filter = new Filter(GuestOSHypervisorVO.class, "hypervisorVersion", true, null, null); + List mappings = listBy(sc, filter); + for (GuestOSHypervisorVO mapping : mappings) { + versions.add(mapping.getHypervisorVersion()); + } + return versions; + } + } diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql b/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql index a868e2a21ea..dd7bfee314c 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql @@ -515,4 +515,7 @@ ALTER VIEW `cloud`.`image_store_view` AS left join `cloud`.`data_center` ON image_store.data_center_id = data_center.id left join - `cloud`.`image_store_details` ON image_store_details.store_id = image_store.id; \ No newline at end of file + `cloud`.`image_store_details` ON image_store_details.store_id = image_store.id; + +-- OVF configured OS while registering deploy-as-is templates +INSERT IGNORE INTO `cloud`.`guest_os` (uuid, category_id, display_name, created) VALUES (UUID(), 11, 'OVF Configured OS', now()); diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/deployasis/DeployAsIsHelperImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/deployasis/DeployAsIsHelperImpl.java index 7e431b1a729..c71fc4da439 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/deployasis/DeployAsIsHelperImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/deployasis/DeployAsIsHelperImpl.java @@ -23,6 +23,7 @@ import com.cloud.agent.api.to.DataStoreTO; import com.cloud.agent.api.to.DataTO; import com.cloud.agent.api.to.DiskTO; import com.cloud.agent.api.to.NicTO; +import com.cloud.agent.api.to.OVFInformationTO; import com.cloud.agent.api.to.deployasis.OVFConfigurationTO; import com.cloud.agent.api.to.deployasis.OVFEulaSectionTO; import com.cloud.agent.api.to.deployasis.OVFPropertyTO; @@ -34,11 +35,24 @@ import com.cloud.deployasis.TemplateDeployAsIsDetailVO; import com.cloud.deployasis.UserVmDeployAsIsDetailVO; import com.cloud.deployasis.dao.TemplateDeployAsIsDetailsDao; import com.cloud.deployasis.dao.UserVmDeployAsIsDetailsDao; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.storage.GuestOSCategoryVO; +import com.cloud.storage.GuestOSHypervisorVO; +import com.cloud.storage.GuestOSVO; import com.cloud.storage.VMTemplateStoragePoolVO; +import com.cloud.storage.VMTemplateVO; import com.cloud.storage.Volume; +import com.cloud.storage.dao.GuestOSCategoryDao; +import com.cloud.storage.dao.GuestOSDao; +import com.cloud.storage.dao.GuestOSHypervisorDao; +import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VMTemplatePoolDao; +import com.cloud.utils.Pair; import com.cloud.utils.compression.CompressionUtil; import com.cloud.utils.crypt.DBEncryptionUtil; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallbackNoReturn; +import com.cloud.utils.db.TransactionStatus; import com.cloud.vm.VirtualMachineProfile; import com.google.gson.Gson; import com.google.gson.GsonBuilder; @@ -47,14 +61,18 @@ import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.ArrayUtils; +import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; import javax.inject.Inject; import java.io.IOException; +import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; @Component public class DeployAsIsHelperImpl implements DeployAsIsHelper { @@ -70,6 +88,14 @@ public class DeployAsIsHelperImpl implements DeployAsIsHelper { private PrimaryDataStoreDao storagePoolDao; @Inject private VMTemplatePoolDao templateStoragePoolDao; + @Inject + private VMTemplateDao templateDao; + @Inject + private GuestOSDao guestOSDao; + @Inject + private GuestOSHypervisorDao guestOSHypervisorDao; + @Inject + private GuestOSCategoryDao guestOSCategoryDao; static { GsonBuilder builder = new GsonBuilder(); @@ -78,30 +104,180 @@ public class DeployAsIsHelperImpl implements DeployAsIsHelper { } public void persistTemplateDeployAsIsDetails(long templateId, DownloadAnswer answer) { - List ovfProperties = answer.getOvfProperties(); - List networkRequirements = answer.getNetworkRequirements(); - OVFVirtualHardwareSectionTO ovfHardwareSection = answer.getOvfHardwareSection(); - List eulaSections = answer.getEulaSections(); + OVFInformationTO ovfInformationTO = answer.getOvfInformationTO(); + if (ovfInformationTO != null) { + List ovfProperties = ovfInformationTO.getProperties(); + List networkRequirements = ovfInformationTO.getNetworks(); + OVFVirtualHardwareSectionTO ovfHardwareSection = ovfInformationTO.getHardwareSection(); + List eulaSections = ovfInformationTO.getEulaSections(); + Pair guestOsInfo = ovfInformationTO.getGuestOsInfo(); - if (CollectionUtils.isNotEmpty(ovfProperties)) { - persistTemplateDeployAsIsInformationTOList(templateId, ovfProperties); - } - if (CollectionUtils.isNotEmpty(networkRequirements)) { - persistTemplateDeployAsIsInformationTOList(templateId, networkRequirements); - } - if (CollectionUtils.isNotEmpty(eulaSections)) { - persistTemplateDeployAsIsInformationTOList(templateId, eulaSections); - } - if (ovfHardwareSection != null) { - if (CollectionUtils.isNotEmpty(ovfHardwareSection.getConfigurations())) { - persistTemplateDeployAsIsInformationTOList(templateId, ovfHardwareSection.getConfigurations()); + if (CollectionUtils.isNotEmpty(ovfProperties)) { + persistTemplateDeployAsIsInformationTOList(templateId, ovfProperties); } - if (CollectionUtils.isNotEmpty(ovfHardwareSection.getCommonHardwareItems())) { - persistTemplateDeployAsIsInformationTOList(templateId, ovfHardwareSection.getCommonHardwareItems()); + if (CollectionUtils.isNotEmpty(networkRequirements)) { + persistTemplateDeployAsIsInformationTOList(templateId, networkRequirements); + } + if (CollectionUtils.isNotEmpty(eulaSections)) { + persistTemplateDeployAsIsInformationTOList(templateId, eulaSections); + } + String minimumHardwareVersion = null; + if (ovfHardwareSection != null) { + if (CollectionUtils.isNotEmpty(ovfHardwareSection.getConfigurations())) { + persistTemplateDeployAsIsInformationTOList(templateId, ovfHardwareSection.getConfigurations()); + } + if (CollectionUtils.isNotEmpty(ovfHardwareSection.getCommonHardwareItems())) { + persistTemplateDeployAsIsInformationTOList(templateId, ovfHardwareSection.getCommonHardwareItems()); + } + minimumHardwareVersion = ovfHardwareSection.getMinimiumHardwareVersion(); + } + if (guestOsInfo != null) { + String osType = guestOsInfo.first(); + String osDescription = guestOsInfo.second(); + LOGGER.info("Guest OS information retrieved from the template: " + osType + " - " + osDescription); + try { + handleGuestOsFromOVFDescriptor(templateId, osType, osDescription, minimumHardwareVersion); + } catch (Exception e) { + LOGGER.error("Error handling the guest OS read from the OVF " + osType, e); + } } } } + /** + * Handle the guest OS read from the OVF and try to match it to an existing guest OS in DB. + * If the guest OS cannot be mapped to an existing guest OS in DB, then create it and create support for hypervisor versions. + * Roll back actions in case of unexpected erros + */ + private void handleGuestOsFromOVFDescriptor(long templateId, String guestOsType, String guestOsDescription, + String minimumHardwareVersion) { + VMTemplateVO template = templateDao.findById(templateId); + Hypervisor.HypervisorType hypervisor = template.getHypervisorType(); + if (hypervisor != Hypervisor.HypervisorType.VMware) { + return; + } + String minimumHypervisorVersion = getMinimumSupportedHypervisorVersionForHardwareVersion(minimumHardwareVersion); + LOGGER.info("Minimum hardware version " + minimumHardwareVersion + " matched to hypervisor version " + minimumHypervisorVersion + ". " + + "Checking guest OS supporting this version"); + + List guestOsMappings = guestOSHypervisorDao.listByOsNameAndHypervisorMinimumVersion(guestOsType, + hypervisor.toString(), minimumHypervisorVersion); + + Transaction.execute(new TransactionCallbackNoReturn() { + @Override + public void doInTransactionWithoutResult(TransactionStatus status) { + if (CollectionUtils.isNotEmpty(guestOsMappings)) { + Pair> result = updateDeployAsIsTemplateToExistingGuestOs(template, guestOsMappings, guestOsDescription); + if (!result.first()) { + // If couldnt find a guest OS matching the display name, create a new guest OS + updateDeployAsIsTemplateToNewGuestOs(template, guestOsType, guestOsDescription, hypervisor, result.second()); + } + } else { + // The guest OS is not present in DB, create a new guest OS entry and mappings for supported versions + List hypervisorVersions = guestOSHypervisorDao.listHypervisorSupportedVersionsFromMinimumVersion( + hypervisor.toString(), minimumHypervisorVersion); + updateDeployAsIsTemplateToNewGuestOs(template, guestOsType, guestOsDescription, hypervisor, hypervisorVersions); + } + } + }); + } + + /** + * Return true if the template guest OS can be updated to an existing guest OS matching the display name to + * the guest OS description from the OVF. (the second element of the returned pair is not used) + * If the guest OS does not exist, return false and the hypervisor versions that the guest OS must support + */ + private Pair> updateDeployAsIsTemplateToExistingGuestOs(VMTemplateVO template, + List guestOsMappings, + String guestOsDescription) { + Set versionsToSupport = new HashSet<>(); + for (GuestOSHypervisorVO mapping : guestOsMappings) { + long guestOsId = mapping.getGuestOsId(); + GuestOSVO guestOs = guestOSDao.findById(guestOsId); + versionsToSupport.add(mapping.getHypervisorVersion()); + if (guestOs.getDisplayName().equalsIgnoreCase(guestOsDescription)) { + LOGGER.info("Found guest OS mapping for OVF read guest OS " + guestOsDescription + + " to hypervisor version " + mapping.getHypervisorVersion()); + LOGGER.info("Updating deploy-as-is template guest OS to " + guestOs.getDisplayName()); + updateTemplateGuestOsId(template, guestOsId); + return new Pair<>(true, versionsToSupport); + } + } + LOGGER.info("Could not map the OVF read guest OS " + guestOsDescription + " to an existing guest OS"); + return new Pair<>(false, versionsToSupport); + } + + /** + * Updates the deploy-as-is template guest OS doing: + * - Create a new guest OS with the guest OS description parsed from the OVF + * - Create mappings for the new guest OS and supported hypervisor versions + * - Update the template guest OS ID to the new guest OS ID + */ + private void updateDeployAsIsTemplateToNewGuestOs(VMTemplateVO template, String guestOsType, String guestOsDescription, + Hypervisor.HypervisorType hypervisor, Collection hypervisorVersions) { + GuestOSVO newGuestOs = createGuestOsEntry(guestOsDescription); + for (String hypervisorVersion : hypervisorVersions) { + LOGGER.info(String.format("Adding a new guest OS mapping for hypervisor: %s version: %s and " + + "guest OS: %s", hypervisor.toString(), hypervisorVersion, guestOsType)); + createGuestOsHypervisorMapping(newGuestOs.getId(), guestOsType, hypervisor.toString(), hypervisorVersion); + } + updateTemplateGuestOsId(template, newGuestOs.getId()); + } + + private void updateTemplateGuestOsId(VMTemplateVO template, long guestOsId) { + template.setGuestOSId(guestOsId); + templateDao.update(template.getId(), template); + } + + /** + * Create a new entry on guest_os_hypervisor + */ + private void createGuestOsHypervisorMapping(long guestOsId, String guestOsType, String hypervisorType, String hypervisorVersion) { + GuestOSHypervisorVO mappingVO = new GuestOSHypervisorVO(); + mappingVO.setGuestOsId(guestOsId); + mappingVO.setHypervisorType(hypervisorType); + mappingVO.setHypervisorVersion(hypervisorVersion); + mappingVO.setGuestOsName(guestOsType); + guestOSHypervisorDao.persist(mappingVO); + } + + /** + * Create new guest OS entry with category 'Other' + */ + private GuestOSVO createGuestOsEntry(String guestOsDescription) { + GuestOSCategoryVO categoryVO = guestOSCategoryDao.findByCategoryName("Other"); + long categoryId = categoryVO != null ? categoryVO.getId() : 7L; + GuestOSVO guestOSVO = new GuestOSVO(); + guestOSVO.setDisplayName(guestOsDescription); + guestOSVO.setCategoryId(categoryId); + return guestOSDao.persist(guestOSVO); + } + + /** + * Minimum VMware hosts supported version is 6.0 + */ + protected String getMinimumSupportedHypervisorVersionForHardwareVersion(String hardwareVersion) { + // From https://kb.vmware.com/s/article/1003746 and https://kb.vmware.com/s/article/2007240 + String hypervisorVersion = "default"; + if (StringUtils.isBlank(hardwareVersion)) { + return hypervisorVersion; + } + String hwVersion = hardwareVersion.replace("vmx-", ""); + try { + int hwVersionNumber = Integer.parseInt(hwVersion); + if (hwVersionNumber <= 11) { + hypervisorVersion = "6.0"; + } else if (hwVersionNumber == 13) { + hypervisorVersion = "6.5"; + } else if (hwVersionNumber >= 14) { + hypervisorVersion = "6.7"; + } + } catch (NumberFormatException e) { + LOGGER.error("Cannot parse hardware version " + hwVersion + " to integer. Using default hypervisor version", e); + } + return hypervisorVersion; + } + @Override public Map getVirtualMachineDeployAsIsProperties(VirtualMachineProfile vm) { Map map = new HashMap<>(); diff --git a/server/src/main/java/com/cloud/template/TemplateAdapterBase.java b/server/src/main/java/com/cloud/template/TemplateAdapterBase.java index d016fed4a2e..45a10a8c4b3 100644 --- a/server/src/main/java/com/cloud/template/TemplateAdapterBase.java +++ b/server/src/main/java/com/cloud/template/TemplateAdapterBase.java @@ -23,6 +23,7 @@ import java.util.Map; import javax.inject.Inject; +import com.cloud.deployasis.DeployAsIsConstants; import com.cloud.storage.upload.params.IsoUploadParams; import com.cloud.storage.upload.params.TemplateUploadParams; import com.cloud.storage.upload.params.UploadParams; @@ -167,6 +168,13 @@ public abstract class TemplateAdapterBase extends AdapterBase implements Templat if (requiresHVM == null) { requiresHVM = true; } + if (deployAsIs) { + GuestOS deployAsIsGuestOs = ApiDBUtils.findGuestOSByDisplayName(DeployAsIsConstants.DEFAULT_GUEST_OS_DEPLOY_AS_IS); + if (s_logger.isDebugEnabled()) { + s_logger.debug("Setting default guest OS for deploy-as-is template while the template registration is not completed"); + } + guestOSId = deployAsIsGuestOs.getId(); + } } if (isExtractable == null) { diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java index d60d7b8fb20..7ce252a8ed7 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java @@ -37,10 +37,7 @@ import java.util.concurrent.Executors; import javax.naming.ConfigurationException; -import com.cloud.agent.api.to.deployasis.OVFEulaSectionTO; -import com.cloud.agent.api.to.deployasis.OVFVirtualHardwareSectionTO; -import com.cloud.agent.api.to.DatadiskTO; -import com.cloud.agent.api.to.deployasis.OVFPropertyTO; +import com.cloud.agent.api.to.OVFInformationTO; import com.cloud.storage.template.Processor; import com.cloud.storage.template.S3TemplateDownloader; import com.cloud.storage.template.TemplateDownloader; @@ -58,7 +55,6 @@ import com.cloud.storage.template.RawImageProcessor; import com.cloud.storage.template.TARProcessor; import com.cloud.storage.template.VhdProcessor; import com.cloud.storage.template.TemplateConstants; -import com.cloud.agent.api.to.deployasis.OVFNetworkTO; import org.apache.cloudstack.storage.command.DownloadCommand; import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType; import org.apache.cloudstack.storage.command.DownloadProgressCommand; @@ -66,7 +62,6 @@ import org.apache.cloudstack.storage.command.DownloadProgressCommand.RequestType import org.apache.cloudstack.storage.NfsMountManagerImpl.PathParser; import org.apache.cloudstack.storage.resource.NfsSecondaryStorageResource; import org.apache.cloudstack.storage.resource.SecondaryStorageResource; -import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; import com.cloud.agent.api.storage.DownloadAnswer; @@ -131,11 +126,7 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager private long templatePhysicalSize; private final long id; private final ResourceType resourceType; - private List ovfProperties; - private List networks; - private List disks; - private OVFVirtualHardwareSectionTO hardwareSection; - private List eulaSections; + private OVFInformationTO ovfInformationTO; public DownloadJob(TemplateDownloader td, String jobId, long id, String tmpltName, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum, String installPathPrefix, ResourceType resourceType) { @@ -231,44 +222,12 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager this.checksum = checksum; } - public List getOvfProperties() { - return ovfProperties; + public OVFInformationTO getOvfInformationTO() { + return ovfInformationTO; } - public void setOvfProperties(List ovfProperties) { - this.ovfProperties = ovfProperties; - } - - public List getNetworks() { - return networks; - } - - public void setNetworks(List networks) { - this.networks = networks; - } - - public List getDisks() { - return disks; - } - - public void setDisks(List disks) { - this.disks = disks; - } - - public void setVirtualHardwareSection(OVFVirtualHardwareSectionTO section) { - this.hardwareSection = section; - } - - public OVFVirtualHardwareSectionTO getVirtualHardwareSection() { - return this.hardwareSection; - } - - public List getEulaSections() { - return eulaSections; - } - - public void setEulaSections(List eulaSections) { - this.eulaSections = eulaSections; + public void setOvfInformationTO(OVFInformationTO ovfInformationTO) { + this.ovfInformationTO = ovfInformationTO; } } @@ -563,18 +522,8 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager } dnld.setTemplatesize(info.virtualSize); dnld.setTemplatePhysicalSize(info.size); - if (CollectionUtils.isNotEmpty(info.ovfProperties)) { - dnld.setOvfProperties(info.ovfProperties); - } - if (CollectionUtils.isNotEmpty(info.networks)) { - dnld.setNetworks(info.networks); - } - if (CollectionUtils.isNotEmpty(info.disks)) { - dnld.setDisks(info.disks); - } - dnld.setVirtualHardwareSection(info.hardwareSection); - if (CollectionUtils.isNotEmpty(info.eulaSections)) { - dnld.setEulaSections(info.eulaSections); + if (info.ovfInformationTO != null) { + dnld.setOvfInformationTO(info.ovfInformationTO); } break; } @@ -876,18 +825,8 @@ public class DownloadManagerImpl extends ManagerBase implements DownloadManager answer = new DownloadAnswer(jobId, getDownloadPct(jobId), getDownloadError(jobId), getDownloadStatus2(jobId), getDownloadLocalPath(jobId), getInstallPath(jobId), getDownloadTemplateSize(jobId), getDownloadTemplatePhysicalSize(jobId), getDownloadCheckSum(jobId)); - if (CollectionUtils.isNotEmpty(dj.getOvfProperties())) { - answer.setOvfProperties(dj.getOvfProperties()); - } - if (CollectionUtils.isNotEmpty(dj.getNetworks())) { - answer.setNetworkRequirements(dj.getNetworks()); - } - if (CollectionUtils.isNotEmpty(dj.getDisks())) { - answer.setDisks(dj.getDisks()); - } - answer.setOvfHardwareSection(dj.getVirtualHardwareSection()); - if (CollectionUtils.isNotEmpty(dj.getEulaSections())) { - answer.setEulaSections(dj.getEulaSections()); + if (dj.getOvfInformationTO() != null) { + answer.setOvfInformationTO(dj.getOvfInformationTO()); } jobs.remove(jobId); return answer; From 7a8dd46b326199db649f2abb54090556970cb0c7 Mon Sep 17 00:00:00 2001 From: nvazquez Date: Fri, 25 Sep 2020 03:32:41 -0300 Subject: [PATCH 103/164] Fix deploy as-is disks cleanup --- .../cloudstack/storage/to/VolumeObjectTO.java | 6 +++++ .../resource/VmwareStorageProcessor.java | 25 ++++++++++++++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java b/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java index e47d13ed669..6d613ab1269 100644 --- a/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java +++ b/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java @@ -62,6 +62,7 @@ public class VolumeObjectTO implements DataTO { private Hypervisor.HypervisorType hypervisorType; private MigrationOptions migrationOptions; private boolean directDownload; + private boolean deployAsIs; public VolumeObjectTO() { @@ -102,6 +103,7 @@ public class VolumeObjectTO implements DataTO { setDeviceId(volume.getDeviceId()); this.migrationOptions = volume.getMigrationOptions(); this.directDownload = volume.isDirectDownload(); + this.deployAsIs = volume.isDeployAsIs(); } public String getUuid() { @@ -313,4 +315,8 @@ public class VolumeObjectTO implements DataTO { public boolean isDirectDownload() { return directDownload; } + + public boolean isDeployAsIs() { + return deployAsIs; + } } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java index 5fc71d379d8..0d026cdddde 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -2474,9 +2474,14 @@ public class VmwareStorageProcessor implements StorageProcessor { List> dynamicTargetsToRemove = null; + boolean deployAsIs = vol.isDeployAsIs(); if (vmMo != null) { if (s_logger.isInfoEnabled()) { - s_logger.info("Destroy root volume and VM itself. vmName " + vmName); + if (deployAsIs) { + s_logger.info("Destroying root volume " + vol.getPath() + " of deploy-as-is VM " + vmName); + } else { + s_logger.info("Destroy root volume and VM itself. vmName " + vmName); + } } VirtualMachineDiskInfo diskInfo = null; @@ -2524,6 +2529,24 @@ public class VmwareStorageProcessor implements StorageProcessor { } } } + } else if (deployAsIs) { + if (s_logger.isInfoEnabled()) { + s_logger.info("Destroying root volume " + vol.getPath() + " of already removed deploy-as-is VM " + vmName); + } + // The disks of the deploy-as-is VM have been detached from the VM and moved to root folder + String deployAsIsRootDiskPath = dsMo.searchFileInSubFolders(vol.getPath() + VmwareResource.VMDK_EXTENSION, + true, null); + if (StringUtils.isNotBlank(deployAsIsRootDiskPath)) { + if (s_logger.isInfoEnabled()) { + s_logger.info("Removing disk " + deployAsIsRootDiskPath); + } + dsMo.deleteFile(deployAsIsRootDiskPath, morDc, true); + String deltaFilePath = dsMo.searchFileInSubFolders(vol.getPath() + "-delta" + VmwareResource.VMDK_EXTENSION, + true, null); + if (StringUtils.isNotBlank(deltaFilePath)) { + dsMo.deleteFile(deltaFilePath, morDc, true); + } + } } /* From bb90ce5bd783687925c0b419582e94ad11957a9a Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Thu, 24 Sep 2020 02:17:18 +0530 Subject: [PATCH 104/164] Fix for dettached disk migrations from vVols to other datastores --- .../vmware/resource/VmwareResource.java | 39 ++++++++++++------- .../resource/VmwareStorageProcessor.java | 10 ++++- 2 files changed, 32 insertions(+), 17 deletions(-) diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 91bebcabaf9..06c0e3133a7 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -4770,6 +4770,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa ManagedObjectReference morSourceDS = null; ManagedObjectReference morDestintionDS = null; String vmdkDataStorePath = null; + boolean isvVolsInvolved = false; String vmName = null; try { @@ -4780,10 +4781,12 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa dsMo = new DatastoreMO(hyperHost.getContext(), morSourceDS); morDestintionDS = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, cmd.getTargetPool().getUuid()); destinationDsMo = new DatastoreMO(hyperHost.getContext(), morDestintionDS); - vmName = getWorkerName(getServiceContext(), cmd, 0, dsMo); - if (destinationDsMo.getDatastoreType().equalsIgnoreCase("VVOL")) + vmName = getWorkerName(getServiceContext(), cmd, 0, dsMo); + if (destinationDsMo.getDatastoreType().equalsIgnoreCase("VVOL")) { + isvVolsInvolved = true; vmName = getWorkerName(getServiceContext(), cmd, 0, destinationDsMo); + } // OfflineVmwareMigration: refactor for re-use // OfflineVmwareMigration: 1. find data(store) @@ -4808,12 +4811,16 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } vmdkDataStorePath = VmwareStorageLayoutHelper.getVmwareDatastorePathFromVmdkFileName(dsMo, path, vmdkFileName); } - if (!dsMo.fileExists(vmdkDataStorePath)) { + if (!dsMo.folderExists(String.format("[%s]", dsMo.getName()), path) || !dsMo.fileExists(vmdkDataStorePath)) { if (s_logger.isDebugEnabled()) { s_logger.debug(String.format("path not found (%s), trying under '%s'", vmdkFileName, vmName)); } vmdkDataStorePath = VmwareStorageLayoutHelper.getVmwareDatastorePathFromVmdkFileName(dsMo, vmName, vmdkFileName); } + if (!dsMo.folderExists(String.format("[%s]", dsMo.getName()), vmName) || !dsMo.fileExists(vmdkDataStorePath)) { + vmdkDataStorePath = dsMo.searchFileInSubFolders(vmdkFileName, true, null); + } + if (s_logger.isDebugEnabled()) { s_logger.debug(String.format("attaching %s to %s for migration", vmdkDataStorePath, vmMo.getVmName())); } @@ -4880,21 +4887,23 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } } if (answer instanceof MigrateVolumeAnswer) { - String newPath = ((MigrateVolumeAnswer) answer).getVolumePath(); - String vmdkFileName = newPath + VMDK_EXTENSION; - try { - VmwareStorageLayoutHelper.syncVolumeToRootFolder(dsMo.getOwnerDatacenter().first(), dsMo, newPath, vmName); - vmdkDataStorePath = VmwareStorageLayoutHelper.getLegacyDatastorePathFromVmdkFileName(dsMo, vmdkFileName); + if (!isvVolsInvolved) { + String newPath = ((MigrateVolumeAnswer) answer).getVolumePath(); + String vmdkFileName = newPath + VMDK_EXTENSION; + try { + VmwareStorageLayoutHelper.syncVolumeToRootFolder(dsMo.getOwnerDatacenter().first(), dsMo, newPath, vmName); + vmdkDataStorePath = VmwareStorageLayoutHelper.getLegacyDatastorePathFromVmdkFileName(dsMo, vmdkFileName); - if (!dsMo.fileExists(vmdkDataStorePath)) { - String msg = String.format("Migration of volume '%s' failed; file (%s) not found as path '%s'", cmd.getVolumePath(), vmdkFileName, vmdkDataStorePath); - s_logger.error(msg); + if (!dsMo.fileExists(vmdkDataStorePath)) { + String msg = String.format("Migration of volume '%s' failed; file (%s) not found as path '%s'", cmd.getVolumePath(), vmdkFileName, vmdkDataStorePath); + s_logger.error(msg); + answer = new Answer(cmd, false, msg); + } + } catch (Exception e) { + String msg = String.format("Migration of volume '%s' failed due to %s", cmd.getVolumePath(), e.getLocalizedMessage()); + s_logger.error(msg, e); answer = new Answer(cmd, false, msg); } - } catch (Exception e) { - String msg = String.format("Migration of volume '%s' failed due to %s", cmd.getVolumePath(), e.getLocalizedMessage()); - s_logger.error(msg, e); - answer = new Answer(cmd, false, msg); } } return answer; diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java index 0d026cdddde..35f77e2e358 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -2091,9 +2091,13 @@ public class VmwareStorageProcessor implements StorageProcessor { if (!dsMo.fileExists(datastoreVolumePath)) { datastoreVolumePath = VmwareStorageLayoutHelper.getVmwareDatastorePathFromVmdkFileName(dsMo, vmName, volumeTO.getPath() + ".vmdk"); } - if (!dsMo.fileExists(datastoreVolumePath)) { + if (!dsMo.folderExists(String.format("[%s]", dsMo.getName()), vmName) || !dsMo.fileExists(datastoreVolumePath)) { + datastoreVolumePath = VmwareStorageLayoutHelper.getVmwareDatastorePathFromVmdkFileName(dsMo, volumeTO.getPath(), volumeTO.getPath() + ".vmdk"); + } + if (!dsMo.folderExists(String.format("[%s]", dsMo.getName()), volumeTO.getPath()) || !dsMo.fileExists(datastoreVolumePath)) { datastoreVolumePath = dsMo.searchFileInSubFolders(volumeTO.getPath() + ".vmdk", true, null); } + } else { datastoreVolumePath = VmwareStorageLayoutHelper.syncVolumeToVmDefaultFolder(dsMo.getOwnerDatacenter().first(), vmName, dsMo, volumeTO.getPath(), VmwareManager.s_vmwareSearchExcludeFolder.value()); } @@ -3592,7 +3596,9 @@ public class VmwareStorageProcessor implements StorageProcessor { throw new Exception("Unable to create container VM for volume creation"); } - clonedVm.moveAllVmDiskFiles(primaryDsMo, HypervisorHostHelper.VSPHERE_DATASTORE_BASE_FOLDER, false); + if(!primaryDsMo.getDatastoreType().equalsIgnoreCase("VVOL")) { + clonedVm.moveAllVmDiskFiles(primaryDsMo, HypervisorHostHelper.VSPHERE_DATASTORE_BASE_FOLDER, false); + } clonedVm.detachAllDisks(); return _storage.getSize(srcOVFFileName); } finally { From 81b184877bc7287d32cd85dc2059e06f3df94e97 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Mon, 28 Sep 2020 11:04:56 +0530 Subject: [PATCH 105/164] Added SesParse support which is an improved version of VMFSsparse virtual disk format --- .../java/com/cloud/hypervisor/vmware/mo/VmdkAdapterType.java | 2 ++ .../com/cloud/hypervisor/vmware/mo/VmdkFileDescriptor.java | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VmdkAdapterType.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VmdkAdapterType.java index f602c46d795..ff0a7d30672 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VmdkAdapterType.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VmdkAdapterType.java @@ -35,6 +35,8 @@ public enum VmdkAdapterType { } public static VmdkAdapterType getType(String vmdkAdapterType) { + if (vmdkAdapterType == null) + return VmdkAdapterType.none; if (vmdkAdapterType.equalsIgnoreCase("ide")) { return VmdkAdapterType.ide; } else if (vmdkAdapterType.equalsIgnoreCase("lsilogic")) { diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VmdkFileDescriptor.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VmdkFileDescriptor.java index 556efd7e7fa..7ede78f1d6f 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VmdkFileDescriptor.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VmdkFileDescriptor.java @@ -31,6 +31,7 @@ public class VmdkFileDescriptor { private static final Logger s_logger = Logger.getLogger(VmdkFileDescriptor.class); private static final String VMDK_PROPERTY_CREATE_TYPE = "createType"; private static final String VMDK_CREATE_TYPE_VMFSSPARSE = "vmfsSparse"; + private static final String VMDK_CREATE_TYPE_SESPARSE = "SEsparse"; private static final String VMDK_PROPERTY_ADAPTER_TYPE = "ddb.adapterType"; private Properties _properties = new Properties(); @@ -89,7 +90,7 @@ public class VmdkFileDescriptor { public boolean isVmfsSparseFile() { String vmdkCreateType = _properties.getProperty(VMDK_PROPERTY_CREATE_TYPE); - if (vmdkCreateType.equalsIgnoreCase(VMDK_CREATE_TYPE_VMFSSPARSE)) { + if (vmdkCreateType.equalsIgnoreCase(VMDK_CREATE_TYPE_VMFSSPARSE) || vmdkCreateType.equalsIgnoreCase(VMDK_CREATE_TYPE_SESPARSE)) { return true; } return false; From 32d85b0fa23e36374e3a221f44420f6b3cb78c92 Mon Sep 17 00:00:00 2001 From: nvazquez Date: Mon, 28 Sep 2020 22:59:48 -0300 Subject: [PATCH 106/164] Display storage on logging when not deploy-as-is and guest OS small refactor --- .../com/cloud/deploy/DeployDestination.java | 14 ++++++-- .../main/java/com/cloud/storage/Volume.java | 2 ++ .../cloud/vm/VirtualMachineManagerImpl.java | 2 +- .../main/java/com/cloud/vm/VmWorkMigrate.java | 9 +++-- .../main/java/com/cloud/storage/VolumeVO.java | 9 +++++ .../META-INF/db/schema-41400to41500.sql | 20 +++++++++-- .../deployasis/DeployAsIsHelperImpl.java | 36 +++---------------- .../deploy/DeploymentPlanningManagerImpl.java | 21 ++++++++--- 8 files changed, 69 insertions(+), 44 deletions(-) diff --git a/api/src/main/java/com/cloud/deploy/DeployDestination.java b/api/src/main/java/com/cloud/deploy/DeployDestination.java index 18503fee74b..91b2068e94b 100644 --- a/api/src/main/java/com/cloud/deploy/DeployDestination.java +++ b/api/src/main/java/com/cloud/deploy/DeployDestination.java @@ -36,6 +36,11 @@ public class DeployDestination implements Serializable { Host _host; Map _storage; + /** + * Display volume <-> storage pool mapping by default + */ + boolean displayStorage = true; + public DataCenter getDataCenter() { return _dc; } @@ -63,9 +68,10 @@ public class DeployDestination implements Serializable { _host = host; } - public DeployDestination(DataCenter dc, Pod pod, Cluster cluster, Host host, Map storage) { + public DeployDestination(DataCenter dc, Pod pod, Cluster cluster, Host host, Map storage, boolean displayStorage) { this(dc, pod, cluster, host); _storage = storage; + this.displayStorage = displayStorage; } public DeployDestination() { @@ -139,7 +145,7 @@ public class DeployDestination implements Serializable { destination.append("Cluster(").append(clusterId).append(")").append("-"); destination.append("Host(").append(hostId).append(")").append("-"); destination.append("Storage("); - if (_storage != null) { + if (displayStorage && _storage != null) { StringBuffer storageBuf = new StringBuffer(); //String storageStr = ""; for (Volume vol : _storage.keySet()) { @@ -160,4 +166,8 @@ public class DeployDestination implements Serializable { } return destination.append(")]").toString(); } + + public boolean isDisplayStorage() { + return displayStorage; + } } diff --git a/api/src/main/java/com/cloud/storage/Volume.java b/api/src/main/java/com/cloud/storage/Volume.java index 5fd78efb307..5979697b555 100644 --- a/api/src/main/java/com/cloud/storage/Volume.java +++ b/api/src/main/java/com/cloud/storage/Volume.java @@ -235,4 +235,6 @@ public interface Volume extends ControlledEntity, Identity, InternalIdentity, Ba boolean isDisplayVolume(); boolean isDisplay(); + + boolean isDeployAsIs(); } diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 8a31556a221..02ceef276bf 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -1150,7 +1150,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac try { resetVmNicsDeviceId(vm.getId()); - _networkMgr.prepare(vmProfile, new DeployDestination(dest.getDataCenter(), dest.getPod(), null, null, dest.getStorageForDisks()), ctx); + _networkMgr.prepare(vmProfile, new DeployDestination(dest.getDataCenter(), dest.getPod(), null, null, dest.getStorageForDisks(), dest.isDisplayStorage()), ctx); if (vm.getHypervisorType() != HypervisorType.BareMetal) { volumeMgr.prepare(vmProfile, dest); } diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmWorkMigrate.java b/engine/orchestration/src/main/java/com/cloud/vm/VmWorkMigrate.java index 5bcea9a8aca..1d7deca2d9e 100644 --- a/engine/orchestration/src/main/java/com/cloud/vm/VmWorkMigrate.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VmWorkMigrate.java @@ -64,14 +64,19 @@ public class VmWorkMigrate extends VmWork { Map vols = null; + boolean displayStorage = true; if (storage != null) { vols = new HashMap(storage.size()); for (Map.Entry entry : storage.entrySet()) { - vols.put(s_entityMgr.findByUuid(Volume.class, entry.getKey()), s_entityMgr.findByUuid(StoragePool.class, entry.getValue())); + Volume volume = s_entityMgr.findByUuid(Volume.class, entry.getKey()); + if (displayStorage && volume.isDeployAsIs()) { + displayStorage = false; + } + vols.put(volume, s_entityMgr.findByUuid(StoragePool.class, entry.getValue())); } } - DeployDestination dest = new DeployDestination(zone, pod, cluster, host, vols); + DeployDestination dest = new DeployDestination(zone, pod, cluster, host, vols, displayStorage); return dest; } diff --git a/engine/schema/src/main/java/com/cloud/storage/VolumeVO.java b/engine/schema/src/main/java/com/cloud/storage/VolumeVO.java index d8323e998f8..b766483e6d1 100644 --- a/engine/schema/src/main/java/com/cloud/storage/VolumeVO.java +++ b/engine/schema/src/main/java/com/cloud/storage/VolumeVO.java @@ -166,6 +166,9 @@ public class VolumeVO implements Volume { @Column(name = "hv_ss_reserve") private Integer hypervisorSnapshotReserve; + @Transient + private boolean deployAsIs; + // Real Constructor public VolumeVO(Type type, String name, long dcId, long domainId, long accountId, long diskOfferingId, Storage.ProvisioningType provisioningType, long size, @@ -261,6 +264,7 @@ public class VolumeVO implements Volume { format = that.getFormat(); provisioningType = that.getProvisioningType(); uuid = UUID.randomUUID().toString(); + deployAsIs = that.isDeployAsIs(); } @Override @@ -572,6 +576,11 @@ public class VolumeVO implements Volume { return displayVolume; } + @Override + public boolean isDeployAsIs() { + return deployAsIs; + } + public void setDisplay(boolean display){ this.displayVolume = display; } diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql b/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql index dd7bfee314c..76907037af1 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql @@ -517,5 +517,21 @@ ALTER VIEW `cloud`.`image_store_view` AS left join `cloud`.`image_store_details` ON image_store_details.store_id = image_store.id; --- OVF configured OS while registering deploy-as-is templates -INSERT IGNORE INTO `cloud`.`guest_os` (uuid, category_id, display_name, created) VALUES (UUID(), 11, 'OVF Configured OS', now()); +-- OVF configured OS while registering deploy-as-is templates Linux 3.x Kernel OS +INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (305, UUID(), 11, 'OVF Configured OS', now()); +INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (306, UUID(), 2, 'Linux 3.x Kernel (64 bit)', now()); +INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (307, UUID(), 2, 'Linux 3.x Kernel (32 bit)', now()); + +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid, hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(), 'VMware', '6.0', 'other3xLinux64Guest', 306, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid, hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(), 'VMware', '6.5', 'other3xLinux64Guest', 306, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid, hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(), 'VMware', '6.7', 'other3xLinux64Guest', 306, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid, hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(), 'VMware', '6.7.1', 'other3xLinux64Guest', 306, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid, hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(), 'VMware', '6.7.2', 'other3xLinux64Guest', 306, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid, hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(), 'VMware', '6.7.3', 'other3xLinux64Guest', 306, now(), 0); + +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid, hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(), 'VMware', '6.0', 'other3xLinuxGuest', 307, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid, hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(), 'VMware', '6.5', 'other3xLinuxGuest', 307, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid, hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(), 'VMware', '6.7', 'other3xLinuxGuest', 307, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid, hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(), 'VMware', '6.7.1', 'other3xLinuxGuest', 307, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid, hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(), 'VMware', '6.7.2', 'other3xLinuxGuest', 307, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid, hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(), 'VMware', '6.7.3', 'other3xLinuxGuest', 307, now(), 0); diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/deployasis/DeployAsIsHelperImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/deployasis/DeployAsIsHelperImpl.java index c71fc4da439..9dde8959152 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/deployasis/DeployAsIsHelperImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/deployasis/DeployAsIsHelperImpl.java @@ -69,10 +69,8 @@ import javax.inject.Inject; import java.io.IOException; import java.util.Collection; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; @Component public class DeployAsIsHelperImpl implements DeployAsIsHelper { @@ -167,11 +165,10 @@ public class DeployAsIsHelperImpl implements DeployAsIsHelper { @Override public void doInTransactionWithoutResult(TransactionStatus status) { if (CollectionUtils.isNotEmpty(guestOsMappings)) { - Pair> result = updateDeployAsIsTemplateToExistingGuestOs(template, guestOsMappings, guestOsDescription); - if (!result.first()) { - // If couldnt find a guest OS matching the display name, create a new guest OS - updateDeployAsIsTemplateToNewGuestOs(template, guestOsType, guestOsDescription, hypervisor, result.second()); - } + GuestOSHypervisorVO mapping = guestOsMappings.get(0); + long guestOsId = mapping.getGuestOsId(); + LOGGER.info("Updating deploy-as-is template guest OS to " + guestOsType); + updateTemplateGuestOsId(template, guestOsId); } else { // The guest OS is not present in DB, create a new guest OS entry and mappings for supported versions List hypervisorVersions = guestOSHypervisorDao.listHypervisorSupportedVersionsFromMinimumVersion( @@ -182,31 +179,6 @@ public class DeployAsIsHelperImpl implements DeployAsIsHelper { }); } - /** - * Return true if the template guest OS can be updated to an existing guest OS matching the display name to - * the guest OS description from the OVF. (the second element of the returned pair is not used) - * If the guest OS does not exist, return false and the hypervisor versions that the guest OS must support - */ - private Pair> updateDeployAsIsTemplateToExistingGuestOs(VMTemplateVO template, - List guestOsMappings, - String guestOsDescription) { - Set versionsToSupport = new HashSet<>(); - for (GuestOSHypervisorVO mapping : guestOsMappings) { - long guestOsId = mapping.getGuestOsId(); - GuestOSVO guestOs = guestOSDao.findById(guestOsId); - versionsToSupport.add(mapping.getHypervisorVersion()); - if (guestOs.getDisplayName().equalsIgnoreCase(guestOsDescription)) { - LOGGER.info("Found guest OS mapping for OVF read guest OS " + guestOsDescription + - " to hypervisor version " + mapping.getHypervisorVersion()); - LOGGER.info("Updating deploy-as-is template guest OS to " + guestOs.getDisplayName()); - updateTemplateGuestOsId(template, guestOsId); - return new Pair<>(true, versionsToSupport); - } - } - LOGGER.info("Could not map the OVF read guest OS " + guestOsDescription + " to an existing guest OS"); - return new Pair<>(false, versionsToSupport); - } - /** * Updates the deploy-as-is template guest OS doing: * - Create a new guest OS with the guest OS description parsed from the OVF diff --git a/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java b/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java index 340d9119de8..eb99a8eefe3 100644 --- a/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java +++ b/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java @@ -320,8 +320,9 @@ StateListener { return null; } + boolean displayStorage = getDisplayStorageFromVmProfile(vmProfile); if (vm.getHypervisorType() == HypervisorType.BareMetal) { - DeployDestination dest = new DeployDestination(dc, pod, cluster, host, new HashMap()); + DeployDestination dest = new DeployDestination(dc, pod, cluster, host, new HashMap(), displayStorage); s_logger.debug("Returning Deployment Destination: " + dest); return dest; } @@ -351,7 +352,7 @@ StateListener { for (Volume vol : readyAndReusedVolumes) { storageVolMap.remove(vol); } - DeployDestination dest = new DeployDestination(dc, pod, cluster, host, storageVolMap); + DeployDestination dest = new DeployDestination(dc, pod, cluster, host, storageVolMap, displayStorage); s_logger.debug("Returning Deployment Destination: " + dest); return dest; } @@ -440,6 +441,7 @@ StateListener { hostHasCapacity = _capacityMgr.checkIfHostHasCapacity(host.getId(), cpu_requested, ram_requested, false, cpuOvercommitRatio, memoryOvercommitRatio, true); } + boolean displayStorage = getDisplayStorageFromVmProfile(vmProfile); if (hostHasCapacity && hostHasCpuCapability) { s_logger.debug("The last host of this VM is UP and has enough capacity"); @@ -449,7 +451,7 @@ StateListener { Pod pod = _podDao.findById(host.getPodId()); Cluster cluster = _clusterDao.findById(host.getClusterId()); if (vm.getHypervisorType() == HypervisorType.BareMetal) { - DeployDestination dest = new DeployDestination(dc, pod, cluster, host, new HashMap()); + DeployDestination dest = new DeployDestination(dc, pod, cluster, host, new HashMap(), displayStorage); s_logger.debug("Returning Deployment Destination: " + dest); return dest; } @@ -482,7 +484,7 @@ StateListener { storageVolMap.remove(vol); } DeployDestination dest = new DeployDestination(dc, pod, cluster, host, - storageVolMap); + storageVolMap, displayStorage); s_logger.debug("Returning Deployment Destination: " + dest); return dest; } @@ -555,6 +557,13 @@ StateListener { return dest; } + /** + * Display storage in the logs by default if the template is not deploy-as-is. + */ + private boolean getDisplayStorageFromVmProfile(VirtualMachineProfile vmProfile) { + return vmProfile == null || vmProfile.getTemplate() == null || !vmProfile.getTemplate().isDeployAsIs(); + } + @Override public DeploymentPlanner getDeploymentPlannerByName(String plannerName) { if (plannerName != null) { @@ -1121,7 +1130,8 @@ StateListener { for (Volume vol : readyAndReusedVolumes) { storageVolMap.remove(vol); } - DeployDestination dest = new DeployDestination(dc, pod, clusterVO, host, storageVolMap); + boolean displayStorage = getDisplayStorageFromVmProfile(vmProfile); + DeployDestination dest = new DeployDestination(dc, pod, clusterVO, host, storageVolMap, displayStorage); s_logger.debug("Returning Deployment Destination: " + dest); return dest; } @@ -1329,6 +1339,7 @@ StateListener { if (hostCanAccessPool && haveEnoughSpace && hostAffinityCheck && checkIfHostFitsPlannerUsage(potentialHost.getId(), resourceUsageRequired)) { s_logger.debug("Found a potential host " + "id: " + potentialHost.getId() + " name: " + potentialHost.getName() + " and associated storage pools for this VM"); + volumeAllocationMap.clear(); return new Pair>(potentialHost, storage); } else { avoid.addHost(potentialHost.getId()); From 667cb53772104387576012c783435dc06865a0b6 Mon Sep 17 00:00:00 2001 From: nvazquez Date: Tue, 29 Sep 2020 00:06:25 -0300 Subject: [PATCH 107/164] Fix mismatch from OVF nic adapter type and expected adapter type string --- .../vmware/resource/VmwareResource.java | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 06c0e3133a7..a7c82914867 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -2229,7 +2229,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa s_logger.info("Prepare NIC device based on NicTO: " + _gson.toJson(nicTo)); String adapterTypeStr = deployAsIs ? - deployAsIsInfo.getNicAdapterMap().get(nicTo.getDeviceId()) : + mapAdapterType(deployAsIsInfo.getNicAdapterMap().get(nicTo.getDeviceId())) : vmSpec.getDetails().get(VmDetailConstants.NIC_ADAPTER); nicDeviceType = VirtualEthernetCardType.valueOf(adapterTypeStr); @@ -2402,6 +2402,19 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } } + private String mapAdapterType(String adapterStringFromOVF) { + if (StringUtils.isBlank(adapterStringFromOVF) || adapterStringFromOVF.equalsIgnoreCase(VirtualEthernetCardType.E1000.toString())) { + return VirtualEthernetCardType.E1000.toString(); + } else if (adapterStringFromOVF.equalsIgnoreCase(VirtualEthernetCardType.PCNet32.toString())) { + return VirtualEthernetCardType.PCNet32.toString(); + } else if (adapterStringFromOVF.equalsIgnoreCase(VirtualEthernetCardType.Vmxnet2.toString())) { + return VirtualEthernetCardType.Vmxnet2.toString(); + } else if (adapterStringFromOVF.equalsIgnoreCase(VirtualEthernetCardType.Vmxnet3.toString())) { + return VirtualEthernetCardType.Vmxnet3.toString(); + } + return VirtualEthernetCardType.E1000.toString(); + } + private int getDisksChangesNumberFromDisksSpec(DiskTO[] disks, boolean deployAsIs) { if (!deployAsIs) { return disks.length; From 78aa3df440e6899724be4f805ddc2c65a46d506c Mon Sep 17 00:00:00 2001 From: nvazquez Date: Tue, 29 Sep 2020 02:35:06 -0300 Subject: [PATCH 108/164] Fix reboot deploy-as-is VMs --- .../com/cloud/hypervisor/guru/VmwareVmImplementer.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java index 5c61887176c..c12c5c20e2d 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java @@ -188,8 +188,12 @@ class VmwareVmImplementer { private void setDeployAsIsInfoTO(VirtualMachineProfile vm, VirtualMachineTO to, Map details) { String configuration = details.getOrDefault(VmDetailConstants.DEPLOY_AS_IS_CONFIGURATION, null); Map properties = deployAsIsHelper.getVirtualMachineDeployAsIsProperties(vm); - String destStoragePool = deployAsIsHelper.getAllocatedVirtualMachineDestinationStoragePool(vm); - String templatePath = deployAsIsHelper.getAllocatedVirtualMachineTemplatePath(vm, configuration, destStoragePool); + String templatePath = null; + String destStoragePool = null; + if (vm.getVirtualMachine().getState() == VirtualMachine.State.Starting) { + destStoragePool = deployAsIsHelper.getAllocatedVirtualMachineDestinationStoragePool(vm); + templatePath = deployAsIsHelper.getAllocatedVirtualMachineTemplatePath(vm, configuration, destStoragePool); + } Map nicsAdapterMapping = deployAsIsHelper.getAllocatedVirtualMachineNicsAdapterMapping(vm, to.getNics()); DeployAsIsInfoTO info = new DeployAsIsInfoTO(templatePath, destStoragePool, properties, nicsAdapterMapping); to.setDeployAsIsInfo(info); From 1638d7243c1e4de30e6e2d22baeda2876d2eee78 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Tue, 29 Sep 2020 15:06:52 +0530 Subject: [PATCH 109/164] Get allocated size for datastoreCluster from its child datastores to show the accumalated value for Datastore cluster --- .../cloud/api/query/dao/StoragePoolJoinDaoImpl.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java index a248d71e72c..d2fe6a55edc 100644 --- a/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java @@ -20,6 +20,7 @@ import com.cloud.api.ApiDBUtils; import com.cloud.api.query.vo.StoragePoolJoinVO; import com.cloud.capacity.CapacityManager; import com.cloud.storage.DataStoreRole; +import com.cloud.storage.Storage; import com.cloud.storage.StoragePool; import com.cloud.storage.StorageStats; import com.cloud.utils.StringUtils; @@ -34,6 +35,7 @@ import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailVO; import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -104,6 +106,15 @@ public class StoragePoolJoinDaoImpl extends GenericDaoBase childDatastores = storagePoolDao.listChildStoragePoolsInDatastoreCluster(pool.getId()); + if (childDatastores != null) { + for (StoragePoolVO childDatastore: childDatastores) { + StoragePoolJoinVO childDSJoinVO = findById(childDatastore.getId()); + allocatedSize += (childDSJoinVO.getUsedCapacity() + childDSJoinVO.getReservedCapacity()); + } + } + } poolResponse.setDiskSizeTotal(pool.getCapacityBytes()); poolResponse.setDiskSizeAllocated(allocatedSize); poolResponse.setCapacityIops(pool.getCapacityIops()); From b0d3168e0b78085036c05245c638b3a0d7eb7750 Mon Sep 17 00:00:00 2001 From: nvazquez Date: Wed, 30 Sep 2020 00:54:31 -0300 Subject: [PATCH 110/164] Fail template registration when guest OS not found --- .../image/BaseImageStoreDriverImpl.java | 10 +- .../image/deployasis/DeployAsIsHelper.java | 3 +- .../deployasis/DeployAsIsHelperImpl.java | 106 +++++++++--------- 3 files changed, 63 insertions(+), 56 deletions(-) diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java index 58c24537d57..37a7985ce6f 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java @@ -204,13 +204,18 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver { if (tmpltStoreVO != null) { if (tmpltStoreVO.getDownloadState() == VMTemplateStorageResourceAssoc.Status.DOWNLOADED) { if (template.isDeployAsIs()) { - deployAsIsHelper.persistTemplateDeployAsIsDetails(template.getId(), answer); + boolean persistDeployAsIs = deployAsIsHelper.persistTemplateDeployAsIsDetails(template.getId(), answer, tmpltStoreVO); + if (!persistDeployAsIs) { + LOGGER.info("Failed persisting deploy-as-is template details for template " + template.getName()); + return null; + } } if (LOGGER.isDebugEnabled()) { LOGGER.debug("Template is already in DOWNLOADED state, ignore further incoming DownloadAnswer"); } return null; } + LOGGER.info("Updating store ref entry for template " + template.getName()); TemplateDataStoreVO updateBuilder = _templateStoreDao.createForUpdate(); updateBuilder.setDownloadPercent(answer.getDownloadPct()); updateBuilder.setDownloadState(answer.getDownloadStatus()); @@ -245,9 +250,6 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver { templateDaoBuilder.setChecksum(answer.getCheckSum()); _templateDao.update(obj.getId(), templateDaoBuilder); } - if (template.isDeployAsIs()) { - deployAsIsHelper.persistTemplateDeployAsIsDetails(template.getId(), answer); - } CreateCmdResult result = new CreateCmdResult(null, null); caller.complete(result); diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/deployasis/DeployAsIsHelper.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/deployasis/DeployAsIsHelper.java index 303161c0c59..4f7277e31cd 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/deployasis/DeployAsIsHelper.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/deployasis/DeployAsIsHelper.java @@ -19,12 +19,13 @@ package org.apache.cloudstack.storage.image.deployasis; import com.cloud.agent.api.storage.DownloadAnswer; import com.cloud.agent.api.to.NicTO; import com.cloud.vm.VirtualMachineProfile; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import java.util.Map; public interface DeployAsIsHelper { - void persistTemplateDeployAsIsDetails(long templateId, DownloadAnswer answer); + boolean persistTemplateDeployAsIsDetails(long templateId, DownloadAnswer answer, TemplateDataStoreVO tmpltStoreVO); Map getVirtualMachineDeployAsIsProperties(VirtualMachineProfile vmId); String getAllocatedVirtualMachineTemplatePath(VirtualMachineProfile vm, String configuration, String destStoragePool); diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/deployasis/DeployAsIsHelperImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/deployasis/DeployAsIsHelperImpl.java index 9dde8959152..0d52f57ac98 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/deployasis/DeployAsIsHelperImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/deployasis/DeployAsIsHelperImpl.java @@ -40,6 +40,7 @@ import com.cloud.storage.GuestOSCategoryVO; import com.cloud.storage.GuestOSHypervisorVO; import com.cloud.storage.GuestOSVO; import com.cloud.storage.VMTemplateStoragePoolVO; +import com.cloud.storage.VMTemplateStorageResourceAssoc; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.Volume; import com.cloud.storage.dao.GuestOSCategoryDao; @@ -50,15 +51,15 @@ import com.cloud.storage.dao.VMTemplatePoolDao; import com.cloud.utils.Pair; import com.cloud.utils.compression.CompressionUtil; import com.cloud.utils.crypt.DBEncryptionUtil; -import com.cloud.utils.db.Transaction; -import com.cloud.utils.db.TransactionCallbackNoReturn; -import com.cloud.utils.db.TransactionStatus; +import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VirtualMachineProfile; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.cloud.agent.api.to.deployasis.OVFNetworkTO; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.StringUtils; @@ -72,6 +73,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import static org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.State.Failed; + @Component public class DeployAsIsHelperImpl implements DeployAsIsHelper { @@ -94,6 +97,8 @@ public class DeployAsIsHelperImpl implements DeployAsIsHelper { private GuestOSHypervisorDao guestOSHypervisorDao; @Inject private GuestOSCategoryDao guestOSCategoryDao; + @Inject + private TemplateDataStoreDao templateDataStoreDao; static { GsonBuilder builder = new GsonBuilder(); @@ -101,45 +106,52 @@ public class DeployAsIsHelperImpl implements DeployAsIsHelper { gson = builder.create(); } - public void persistTemplateDeployAsIsDetails(long templateId, DownloadAnswer answer) { - OVFInformationTO ovfInformationTO = answer.getOvfInformationTO(); - if (ovfInformationTO != null) { - List ovfProperties = ovfInformationTO.getProperties(); - List networkRequirements = ovfInformationTO.getNetworks(); - OVFVirtualHardwareSectionTO ovfHardwareSection = ovfInformationTO.getHardwareSection(); - List eulaSections = ovfInformationTO.getEulaSections(); - Pair guestOsInfo = ovfInformationTO.getGuestOsInfo(); + public boolean persistTemplateDeployAsIsDetails(long templateId, DownloadAnswer answer, TemplateDataStoreVO tmpltStoreVO) { + try { + OVFInformationTO ovfInformationTO = answer.getOvfInformationTO(); + if (ovfInformationTO != null) { + List ovfProperties = ovfInformationTO.getProperties(); + List networkRequirements = ovfInformationTO.getNetworks(); + OVFVirtualHardwareSectionTO ovfHardwareSection = ovfInformationTO.getHardwareSection(); + List eulaSections = ovfInformationTO.getEulaSections(); + Pair guestOsInfo = ovfInformationTO.getGuestOsInfo(); - if (CollectionUtils.isNotEmpty(ovfProperties)) { - persistTemplateDeployAsIsInformationTOList(templateId, ovfProperties); - } - if (CollectionUtils.isNotEmpty(networkRequirements)) { - persistTemplateDeployAsIsInformationTOList(templateId, networkRequirements); - } - if (CollectionUtils.isNotEmpty(eulaSections)) { - persistTemplateDeployAsIsInformationTOList(templateId, eulaSections); - } - String minimumHardwareVersion = null; - if (ovfHardwareSection != null) { - if (CollectionUtils.isNotEmpty(ovfHardwareSection.getConfigurations())) { - persistTemplateDeployAsIsInformationTOList(templateId, ovfHardwareSection.getConfigurations()); + if (CollectionUtils.isNotEmpty(ovfProperties)) { + persistTemplateDeployAsIsInformationTOList(templateId, ovfProperties); } - if (CollectionUtils.isNotEmpty(ovfHardwareSection.getCommonHardwareItems())) { - persistTemplateDeployAsIsInformationTOList(templateId, ovfHardwareSection.getCommonHardwareItems()); + if (CollectionUtils.isNotEmpty(networkRequirements)) { + persistTemplateDeployAsIsInformationTOList(templateId, networkRequirements); } - minimumHardwareVersion = ovfHardwareSection.getMinimiumHardwareVersion(); - } - if (guestOsInfo != null) { - String osType = guestOsInfo.first(); - String osDescription = guestOsInfo.second(); - LOGGER.info("Guest OS information retrieved from the template: " + osType + " - " + osDescription); - try { + if (CollectionUtils.isNotEmpty(eulaSections)) { + persistTemplateDeployAsIsInformationTOList(templateId, eulaSections); + } + String minimumHardwareVersion = null; + if (ovfHardwareSection != null) { + if (CollectionUtils.isNotEmpty(ovfHardwareSection.getConfigurations())) { + persistTemplateDeployAsIsInformationTOList(templateId, ovfHardwareSection.getConfigurations()); + } + if (CollectionUtils.isNotEmpty(ovfHardwareSection.getCommonHardwareItems())) { + persistTemplateDeployAsIsInformationTOList(templateId, ovfHardwareSection.getCommonHardwareItems()); + } + minimumHardwareVersion = ovfHardwareSection.getMinimiumHardwareVersion(); + } + if (guestOsInfo != null) { + String osType = guestOsInfo.first(); + String osDescription = guestOsInfo.second(); + LOGGER.info("Guest OS information retrieved from the template: " + osType + " - " + osDescription); handleGuestOsFromOVFDescriptor(templateId, osType, osDescription, minimumHardwareVersion); - } catch (Exception e) { - LOGGER.error("Error handling the guest OS read from the OVF " + osType, e); } } + } catch (Exception e) { + LOGGER.error("Error persisting deploy-as-is details for template " + templateId, e); + tmpltStoreVO.setErrorString(e.getMessage()); + tmpltStoreVO.setState(Failed); + tmpltStoreVO.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR); + templateDataStoreDao.update(tmpltStoreVO.getId(), tmpltStoreVO); + return false; } + LOGGER.info("Successfully persisted deploy-as-is details for template " + templateId); + return true; } /** @@ -161,22 +173,14 @@ public class DeployAsIsHelperImpl implements DeployAsIsHelper { List guestOsMappings = guestOSHypervisorDao.listByOsNameAndHypervisorMinimumVersion(guestOsType, hypervisor.toString(), minimumHypervisorVersion); - Transaction.execute(new TransactionCallbackNoReturn() { - @Override - public void doInTransactionWithoutResult(TransactionStatus status) { - if (CollectionUtils.isNotEmpty(guestOsMappings)) { - GuestOSHypervisorVO mapping = guestOsMappings.get(0); - long guestOsId = mapping.getGuestOsId(); - LOGGER.info("Updating deploy-as-is template guest OS to " + guestOsType); - updateTemplateGuestOsId(template, guestOsId); - } else { - // The guest OS is not present in DB, create a new guest OS entry and mappings for supported versions - List hypervisorVersions = guestOSHypervisorDao.listHypervisorSupportedVersionsFromMinimumVersion( - hypervisor.toString(), minimumHypervisorVersion); - updateDeployAsIsTemplateToNewGuestOs(template, guestOsType, guestOsDescription, hypervisor, hypervisorVersions); - } - } - }); + if (CollectionUtils.isNotEmpty(guestOsMappings)) { + GuestOSHypervisorVO mapping = guestOsMappings.get(0); + long guestOsId = mapping.getGuestOsId(); + LOGGER.info("Updating deploy-as-is template guest OS to " + guestOsType); + updateTemplateGuestOsId(template, guestOsId); + } else { + throw new CloudRuntimeException("Did not find a guest OS with type " + guestOsType); + } } /** From 6e81efa2c97c37834368600fd62407c3d6677e67 Mon Sep 17 00:00:00 2001 From: nvazquez Date: Wed, 30 Sep 2020 01:24:43 -0300 Subject: [PATCH 111/164] Start addressing review comments --- .../java/com/cloud/agent/api/to/DeployAsIsInfoTO.java | 3 +++ .../java/com/cloud/agent/api/to/OVFInformationTO.java | 3 +++ .../agent/api/to/deployasis/OVFConfigurationTO.java | 3 +++ .../api/to/deployasis/OVFVirtualHardwareItemTO.java | 10 ++++++++-- .../api/command/user/template/ListTemplatesCmd.java | 3 ++- .../cloudstack/api/response/TemplateResponse.java | 6 ++++-- 6 files changed, 23 insertions(+), 5 deletions(-) diff --git a/api/src/main/java/com/cloud/agent/api/to/DeployAsIsInfoTO.java b/api/src/main/java/com/cloud/agent/api/to/DeployAsIsInfoTO.java index d82a2975eaf..ce78f83fb4d 100644 --- a/api/src/main/java/com/cloud/agent/api/to/DeployAsIsInfoTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/DeployAsIsInfoTO.java @@ -21,6 +21,9 @@ import com.cloud.agent.api.LogLevel; import java.util.HashMap; import java.util.Map; +/** + * TO class sent to the hypervisor layer with the information needed to handle deploy-as-is VM deployments + */ public class DeployAsIsInfoTO { private String templatePath; diff --git a/api/src/main/java/com/cloud/agent/api/to/OVFInformationTO.java b/api/src/main/java/com/cloud/agent/api/to/OVFInformationTO.java index 412d362441e..6c6c61d8e95 100644 --- a/api/src/main/java/com/cloud/agent/api/to/OVFInformationTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/OVFInformationTO.java @@ -27,6 +27,9 @@ import com.cloud.utils.Pair; import java.util.List; +/** + * Placeholder class for all the subclasses obtained from the OVF parsing + */ public class OVFInformationTO { @LogLevel(LogLevel.Log4jLevel.Off) diff --git a/api/src/main/java/com/cloud/agent/api/to/deployasis/OVFConfigurationTO.java b/api/src/main/java/com/cloud/agent/api/to/deployasis/OVFConfigurationTO.java index f3cb75089ab..aa3c603f3d0 100644 --- a/api/src/main/java/com/cloud/agent/api/to/deployasis/OVFConfigurationTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/deployasis/OVFConfigurationTO.java @@ -20,6 +20,9 @@ package com.cloud.agent.api.to.deployasis; import java.util.List; +/** + * This class represents a template deployment option (configuration) parsed from the OVF + */ public class OVFConfigurationTO implements TemplateDeployAsIsInformationTO { private final String id; diff --git a/api/src/main/java/com/cloud/agent/api/to/deployasis/OVFVirtualHardwareItemTO.java b/api/src/main/java/com/cloud/agent/api/to/deployasis/OVFVirtualHardwareItemTO.java index 52577506653..f178c23e326 100644 --- a/api/src/main/java/com/cloud/agent/api/to/deployasis/OVFVirtualHardwareItemTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/deployasis/OVFVirtualHardwareItemTO.java @@ -16,10 +16,16 @@ // under the License. package com.cloud.agent.api.to.deployasis; -// From: https://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData.xsd +/** + * A hardware item class representing a hardware item read from the OVF. + * From: https://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_ResourceAllocationSettingData.xsd + */ public class OVFVirtualHardwareItemTO implements TemplateDeployAsIsInformationTO{ - //From: https://schemas.dmtf.org/wbem/cim-html/2/CIM_ResourceAllocationSettingData.html + /** + * The hardware item type + * From: https://schemas.dmtf.org/wbem/cim-html/2/CIM_ResourceAllocationSettingData.html + */ public enum HardwareResourceType { Other("Other", 1), ComputerSystem ("Computer System", 2), diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/template/ListTemplatesCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/template/ListTemplatesCmd.java index 157118285fc..5ef66c50b98 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/template/ListTemplatesCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/template/ListTemplatesCmd.java @@ -89,7 +89,8 @@ public class ListTemplatesCmd extends BaseListTaggedResourcesCmd implements User @Parameter(name = ApiConstants.DETAILS, type = CommandType.LIST, collectionType = CommandType.STRING, - description = "comma separated list of template details requested, value can be a list of [ all, resource, min]") + since = "4.15", + description = "comma separated list of template details requested, value can be a list of [ all, min]") private List viewDetails; ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/response/TemplateResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/TemplateResponse.java index c733f37a9fb..6e699230165 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/TemplateResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/TemplateResponse.java @@ -196,11 +196,13 @@ public class TemplateResponse extends BaseResponseWithTagInformation implements private Boolean directDownload; @SerializedName(ApiConstants.DEPLOY_AS_IS) - @Param(description = "VMware only: true if template is deployed without orchestrating disks and networks but \"as-is\" defined in the template.") + @Param(description = "VMware only: true if template is deployed without orchestrating disks and networks but \"as-is\" defined in the template.", + since = "4.15") private Boolean deployAsIs; @SerializedName(ApiConstants.DEPLOY_AS_IS_DETAILS) - @Param(description = "VMware only: additional key/value details tied with deploy-as-is template") + @Param(description = "VMware only: additional key/value details tied with deploy-as-is template", + since = "4.15") private Map deployAsIsDetails; @SerializedName("parenttemplateid") From 9543fd6e6abd976fd1cc2d49b913daf5431fa7b2 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Tue, 29 Sep 2020 00:49:35 +0530 Subject: [PATCH 112/164] Fix startcommand on Datastore cluster when the volume datastore in CloudStack mismatches with vCenter datastore. Volume could have migrated with in datastore cluster which caused the mismatch Fix dettach volume when volume is not on CloudStack intended datastore --- .../storage/to/PrimaryDataStoreTO.java | 6 +- .../cloudstack/storage/to/VolumeObjectTO.java | 10 ++ .../service/VolumeOrchestrationService.java | 2 +- .../cloud/vm/VirtualMachineManagerImpl.java | 4 +- .../orchestration/VolumeOrchestrator.java | 20 ++- .../datastore/db/PrimaryDataStoreDao.java | 2 + .../datastore/db/PrimaryDataStoreDaoImpl.java | 12 ++ .../vmware/resource/VmwareResource.java | 164 ++++++++++-------- .../resource/VmwareStorageProcessor.java | 61 +++++++ .../cloud/storage/VolumeApiServiceImpl.java | 27 +++ .../vmware/mo/VirtualMachineMO.java | 2 +- 11 files changed, 233 insertions(+), 77 deletions(-) diff --git a/core/src/main/java/org/apache/cloudstack/storage/to/PrimaryDataStoreTO.java b/core/src/main/java/org/apache/cloudstack/storage/to/PrimaryDataStoreTO.java index 1572efe621a..7dab8d9da03 100644 --- a/core/src/main/java/org/apache/cloudstack/storage/to/PrimaryDataStoreTO.java +++ b/core/src/main/java/org/apache/cloudstack/storage/to/PrimaryDataStoreTO.java @@ -39,7 +39,7 @@ public class PrimaryDataStoreTO implements DataStoreTO { public static final String REMOVE_AFTER_COPY = PrimaryDataStore.REMOVE_AFTER_COPY; public static final String VOLUME_SIZE = PrimaryDataStore.VOLUME_SIZE; - private final String uuid; + private String uuid; private final String name; private String type; private final long id; @@ -75,6 +75,10 @@ public class PrimaryDataStoreTO implements DataStoreTO { return this.uuid; } + public void setUuid(String uuid) { + this.uuid = uuid; + } + @Override public String getUrl() { return this.url; diff --git a/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java b/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java index 6d613ab1269..f7e17f7bc14 100644 --- a/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java +++ b/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java @@ -63,6 +63,7 @@ public class VolumeObjectTO implements DataTO { private MigrationOptions migrationOptions; private boolean directDownload; private boolean deployAsIs; + private String updatedDataStoreUUID; public VolumeObjectTO() { @@ -319,4 +320,13 @@ public class VolumeObjectTO implements DataTO { public boolean isDeployAsIs() { return deployAsIs; } + + public String getUpdatedDataStoreUUID() { + return updatedDataStoreUUID; + } + + public void setUpdatedDataStoreUUID(String updatedDataStoreUUID) { + this.updatedDataStoreUUID = updatedDataStoreUUID; + } + } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java index 9baea60f29f..44e993f53b2 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java @@ -134,7 +134,7 @@ public interface VolumeOrchestrationService { StoragePool findStoragePool(DiskProfile dskCh, DataCenter dc, Pod pod, Long clusterId, Long hostId, VirtualMachine vm, Set avoid); - void updateVolumeDiskChain(long volumeId, String path, String chainInfo); + void updateVolumeDiskChain(long volumeId, String path, String chainInfo, String updatedDataStoreUUID); /** * Imports an existing volume for a VM into database. Useful while ingesting an unmanaged VM. diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 02ceef276bf..e8a96f1d6fb 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -1491,9 +1491,9 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac // Before doing this, in a certain situation, getPath() from VolumeObjectTO // returned null instead of an actual path (because it was out of date with the DB). if(vol.getPath() != null) { - volumeMgr.updateVolumeDiskChain(vol.getId(), vol.getPath(), vol.getChainInfo()); + volumeMgr.updateVolumeDiskChain(vol.getId(), vol.getPath(), vol.getChainInfo(), vol.getUpdatedDataStoreUUID()); } else { - volumeMgr.updateVolumeDiskChain(vol.getId(), volume.getPath(), vol.getChainInfo()); + volumeMgr.updateVolumeDiskChain(vol.getId(), volume.getPath(), vol.getChainInfo(), vol.getUpdatedDataStoreUUID()); } } } diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java index a51aae7c8f4..b4d662d449c 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java @@ -1238,9 +1238,15 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati details.put(DiskTO.MOUNT_POINT, volumeInfo.get_iScsiName()); VolumeVO volume = _volumeDao.findById(volumeInfo.getId()); - details.put(DiskTO.PROTOCOL_TYPE, (volume.getPoolType() != null) ? volume.getPoolType().toString() : null); + if (volume.getPoolId() != null) { + StoragePoolVO poolVO = _storagePoolDao.findById(volume.getPoolId()); + if (poolVO.getParent() != 0L) { + details.put(DiskTO.PROTOCOL_TYPE, Storage.StoragePoolType.DatastoreCluster.toString()); + } + } + ChapInfo chapInfo = volService.getChapInfo(volumeInfo, dataStore); if (chapInfo != null) { @@ -1704,7 +1710,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati } @Override - public void updateVolumeDiskChain(long volumeId, String path, String chainInfo) { + public void updateVolumeDiskChain(long volumeId, String path, String chainInfo, String updatedDataStoreUUID) { VolumeVO vol = _volsDao.findById(volumeId); boolean needUpdate = false; // Volume path is not getting updated in the DB, need to find reason and fix the issue. @@ -1719,10 +1725,20 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati needUpdate = true; } + if (updatedDataStoreUUID != null) { + needUpdate = true; + } + if (needUpdate) { s_logger.info("Update volume disk chain info. vol: " + vol.getId() + ", " + vol.getPath() + " -> " + path + ", " + vol.getChainInfo() + " -> " + chainInfo); vol.setPath(path); vol.setChainInfo(chainInfo); + if (updatedDataStoreUUID != null) { + List pools = _storagePoolDao.listPoolsByLikePath(updatedDataStoreUUID); + if (pools != null && !pools.isEmpty()) { + vol.setPoolId(pools.get(0).getId()); + } + } _volsDao.update(volumeId, vol); } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java index 5712411a48e..0cbd0e14f28 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java @@ -127,4 +127,6 @@ public interface PrimaryDataStoreDao extends GenericDao { List listChildStoragePoolsInDatastoreCluster(long poolId); Integer countAll(); + + public List listPoolsByLikePath(String path); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java index 1d1e0a0198d..315f698c099 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java @@ -56,6 +56,7 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase private final SearchBuilder DeleteLvmSearch; private final SearchBuilder DcLocalStorageSearch; private final GenericSearchBuilder StatusCountSearch; + private final SearchBuilder PathLikeSearch; @Inject private StoragePoolDetailsDao _detailsDao; @@ -132,6 +133,17 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase DcLocalStorageSearch.and("path", DcLocalStorageSearch.entity().getPath(), SearchCriteria.Op.EQ); DcLocalStorageSearch.and("scope", DcLocalStorageSearch.entity().getScope(), SearchCriteria.Op.EQ); DcLocalStorageSearch.done(); + + PathLikeSearch = createSearchBuilder(); + PathLikeSearch.and("path", PathLikeSearch.entity().getPath(), SearchCriteria.Op.LIKE); + PathLikeSearch.done(); + } + + @Override + public List listPoolsByLikePath(String path) { + SearchCriteria sc = PathLikeSearch.create(); + sc.setParameters("path", "%" + path + "%"); + return listBy(sc); } @Override diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index a7c82914867..a3990bb4f90 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -1791,13 +1791,6 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa throw new Exception(msg); } - DatastoreMO dsRootVolumeIsOn = getDatastoreThatRootDiskIsOn(dataStoresDetails, disks); - if (dsRootVolumeIsOn == null) { - String msg = "Unable to locate datastore details of root volume"; - s_logger.error(msg); - throw new Exception(msg); - } - VirtualMachineDiskInfoBuilder diskInfoBuilder = null; VirtualDevice[] nicDevices = null; VirtualMachineMO vmMo = hyperHost.findVmOnHyperHost(vmInternalCSName); @@ -1806,12 +1799,14 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa int numScsiControllerForSystemVm = 1; boolean hasSnapshot = false; + List> diskDatastores = null; if (vmMo != null) { s_logger.info("VM " + vmInternalCSName + " already exists, tear down devices for reconfiguration"); if (getVmPowerState(vmMo) != PowerState.PowerOff) vmMo.safePowerOff(_shutdownWaitMs); // retrieve disk information before we tear down + diskDatastores = vmMo.getAllDiskDatastores(); diskInfoBuilder = vmMo.getDiskInfoBuilder(); hasSnapshot = vmMo.hasSnapshot(); nicDevices = vmMo.getNicDevices(); @@ -1836,6 +1831,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa diskInfoBuilder = vmMo.getDiskInfoBuilder(); hasSnapshot = vmMo.hasSnapshot(); + diskDatastores = vmMo.getAllDiskDatastores(); tearDownVmDevices(vmMo, hasSnapshot, deployAsIs); ensureDiskControllersInternal(vmMo, systemVm, controllerInfo, systemVmScsiControllerType, @@ -1866,32 +1862,33 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } mapSpecDisksToClonedDisks(vmMo, vmInternalCSName, specDisks); } else { - Pair rootDiskDataStoreDetails = null; + DiskTO rootDisk = null; for (DiskTO vol : disks) { if (vol.getType() == Volume.Type.ROOT) { - Map details = vol.getDetails(); - boolean managed = false; - - if (details != null) { - managed = Boolean.parseBoolean(details.get(DiskTO.MANAGED)); - } - - if (managed) { - String datastoreName = VmwareResource.getDatastoreName(details.get(DiskTO.IQN)); - - rootDiskDataStoreDetails = dataStoresDetails.get(datastoreName); - } else { - DataStoreTO primaryStore = vol.getData().getDataStore(); - - rootDiskDataStoreDetails = dataStoresDetails.get(primaryStore.getUuid()); + rootDisk = vol; + } + } + Pair rootDiskDataStoreDetails = getDatastoreThatDiskIsOn(dataStoresDetails, rootDisk); + assert (vmSpec.getMinSpeed() != null) && (rootDiskDataStoreDetails != null); + DatastoreMO dsRootVolumeIsOn = rootDiskDataStoreDetails.second(); + if (dsRootVolumeIsOn == null) { + String msg = "Unable to locate datastore details of root volume"; + s_logger.error(msg); + throw new Exception(msg); + } + if (rootDisk.getDetails().get(DiskTO.PROTOCOL_TYPE) != null && rootDisk.getDetails().get(DiskTO.PROTOCOL_TYPE).equalsIgnoreCase("DatastoreCluster")) { + if (diskInfoBuilder != null) { + DatastoreMO diskDatastoreMofromVM = getDataStoreWhereDiskExists(hyperHost, context, diskInfoBuilder, rootDisk, diskDatastores); + if (diskDatastoreMofromVM != null) { + String actualPoolUuid = diskDatastoreMofromVM.getCustomFieldValue(CustomFieldConstants.CLOUD_UUID); + if (!actualPoolUuid.equalsIgnoreCase(rootDisk.getData().getDataStore().getUuid())) { + dsRootVolumeIsOn = diskDatastoreMofromVM; + } } } } - assert (vmSpec.getMinSpeed() != null) && (rootDiskDataStoreDetails != null); - - boolean vmFolderExists = rootDiskDataStoreDetails.second().folderExists(String.format("[%s]", rootDiskDataStoreDetails.second().getName()), vmNameOnVcenter); - String vmxFileFullPath = dsRootVolumeIsOn.searchFileInSubFolders(vmNameOnVcenter + ".vmx", false, VmwareManager.s_vmwareSearchExcludeFolder.value()); + boolean vmFolderExists = dsRootVolumeIsOn.folderExists(String.format("[%s]", dsRootVolumeIsOn.getName()), vmNameOnVcenter); String vmxFileFullPath = dsRootVolumeIsOn.searchFileInSubFolders(vmNameOnVcenter + ".vmx", false, VmwareManager.s_vmwareSearchExcludeFolder.value()); if (vmFolderExists && vmxFileFullPath != null) { // VM can be registered only if .vmx is present. registerVm(vmNameOnVcenter, dsRootVolumeIsOn); vmMo = hyperHost.findVmOnHyperHost(vmInternalCSName); @@ -2121,13 +2118,30 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa iScsiName = details.get(DiskTO.IQN); } + String primaryStoreUuid = primaryStore.getUuid(); // if the storage is managed, iScsiName should not be null - String datastoreName = managed ? VmwareResource.getDatastoreName(iScsiName) : primaryStore.getUuid(); + String datastoreName = managed ? VmwareResource.getDatastoreName(iScsiName) : primaryStoreUuid; Pair volumeDsDetails = dataStoresDetails.get(datastoreName); assert (volumeDsDetails != null); + if (volumeDsDetails == null) { + throw new Exception("Primary datastore " + primaryStore.getUuid() + " is not mounted on host."); + } - String[] diskChain = syncDiskChain(dcMo, vmMo, vmSpec, vol, matchingExistingDisk, dataStoresDetails); + if (vol.getDetails().get(DiskTO.PROTOCOL_TYPE) != null && vol.getDetails().get(DiskTO.PROTOCOL_TYPE).equalsIgnoreCase("DatastoreCluster")) { + if (diskInfoBuilder != null && matchingExistingDisk == null) { + DatastoreMO diskDatastoreMofromVM = getDataStoreWhereDiskExists(hyperHost, context, diskInfoBuilder, vol, diskDatastores); + if (diskDatastoreMofromVM != null) { + String actualPoolUuid = diskDatastoreMofromVM.getCustomFieldValue(CustomFieldConstants.CLOUD_UUID); + if (!actualPoolUuid.equalsIgnoreCase(primaryStore.getUuid())) { + volumeDsDetails = new Pair<>(diskDatastoreMofromVM.getMor(), diskDatastoreMofromVM); + ((PrimaryDataStoreTO)primaryStore).setUuid(actualPoolUuid); + } + } + } + } + + String[] diskChain = syncDiskChain(dcMo, vmMo, vol, matchingExistingDisk, volumeDsDetails.second()); int deviceNumber = -1; if (controllerKey == vmMo.getIDEControllerKey(ideUnitNumber)) { @@ -2872,31 +2886,18 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } // return the finalized disk chain for startup, from top to bottom - private String[] syncDiskChain(DatacenterMO dcMo, VirtualMachineMO vmMo, VirtualMachineTO vmSpec, DiskTO vol, VirtualMachineDiskInfo diskInfo, - HashMap> dataStoresDetails) throws Exception { + private String[] syncDiskChain(DatacenterMO dcMo, VirtualMachineMO vmMo, DiskTO vol, VirtualMachineDiskInfo diskInfo, + DatastoreMO dsMo) throws Exception { VolumeObjectTO volumeTO = (VolumeObjectTO) vol.getData(); - DataStoreTO primaryStore = volumeTO.getDataStore(); Map details = vol.getDetails(); boolean isManaged = false; - String iScsiName = null; if (details != null) { isManaged = Boolean.parseBoolean(details.get(DiskTO.MANAGED)); - iScsiName = details.get(DiskTO.IQN); } - // if the storage is managed, iScsiName should not be null - String datastoreName = isManaged ? VmwareResource.getDatastoreName(iScsiName) : primaryStore.getUuid(); - Pair volumeDsDetails = dataStoresDetails.get(datastoreName); - - if (volumeDsDetails == null) { - throw new Exception("Primary datastore " + primaryStore.getUuid() + " is not mounted on host."); - } - - DatastoreMO dsMo = volumeDsDetails.second(); String datastoreDiskPath; - if (dsMo.getDatastoreType().equalsIgnoreCase("VVOL")) { datastoreDiskPath = VmwareStorageLayoutHelper.getLegacyDatastorePathFromVmdkFileName(dsMo, volumeTO.getPath() + ".vmdk"); if (!dsMo.fileExists(datastoreDiskPath)) { @@ -3327,6 +3328,8 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa volInSpec.setPath(datastoreVolumePath); } else { volInSpec.setPath(file.getFileBaseName()); + if (!file.getDatastoreName().equals(volumeTO.getDataStore().getUuid())) + volInSpec.setUpdatedDataStoreUUID(file.getDatastoreName()); } volInSpec.setChainInfo(_gson.toJson(diskInfo)); } @@ -3462,6 +3465,41 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa return path.substring(0, endIndex).trim(); } + private DatastoreMO getDataStoreWhereDiskExists(VmwareHypervisorHost hyperHost, VmwareContext context, + VirtualMachineDiskInfoBuilder diskInfoBuilder, DiskTO disk, List> diskDatastores) throws Exception { + VolumeObjectTO volume = (VolumeObjectTO) disk.getData(); + String diskBackingFileBaseName = volume.getPath(); + for (Pair diskDatastore : diskDatastores) { + DatastoreMO dsMo = new DatastoreMO(hyperHost.getContext(), diskDatastore.second()); + String dsName = dsMo.getName(); + + VirtualMachineDiskInfo diskInfo = diskInfoBuilder.getDiskInfoByBackingFileBaseName(diskBackingFileBaseName, dsName); + if (diskInfo != null) { + s_logger.info("Found existing disk info from volume path: " + volume.getPath()); + return dsMo; + } else { + String chainInfo = volume.getChainInfo(); + if (chainInfo != null) { + VirtualMachineDiskInfo infoInChain = _gson.fromJson(chainInfo, VirtualMachineDiskInfo.class); + if (infoInChain != null) { + String[] disks = infoInChain.getDiskChain(); + if (disks.length > 0) { + for (String diskPath : disks) { + DatastoreFile file = new DatastoreFile(diskPath); + diskInfo = diskInfoBuilder.getDiskInfoByBackingFileBaseName(file.getFileBaseName(), dsName); + if (diskInfo != null) { + s_logger.info("Found existing disk from chain info: " + diskPath); + return dsMo; + } + } + } + } + } + } + } + return null; + } + private HashMap> inferDatastoreDetailsFromDiskInfo(VmwareHypervisorHost hyperHost, VmwareContext context, DiskTO[] disks, Command cmd) throws Exception { HashMap> mapIdToMors = new HashMap<>(); @@ -3534,39 +3572,25 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa return mapIdToMors; } - private DatastoreMO getDatastoreThatRootDiskIsOn(HashMap> dataStoresDetails, DiskTO disks[]) { + private Pair getDatastoreThatDiskIsOn(HashMap> dataStoresDetails, DiskTO vol) { Pair rootDiskDataStoreDetails = null; - for (DiskTO vol : disks) { - if (vol.getType() == Volume.Type.ROOT) { - Map details = vol.getDetails(); - boolean managed = false; + Map details = vol.getDetails(); + boolean managed = false; - if (details != null) { - managed = Boolean.parseBoolean(details.get(DiskTO.MANAGED)); - } - - if (managed) { - String datastoreName = VmwareResource.getDatastoreName(details.get(DiskTO.IQN)); - - rootDiskDataStoreDetails = dataStoresDetails.get(datastoreName); - - break; - } else { - DataStoreTO primaryStore = vol.getData().getDataStore(); - - rootDiskDataStoreDetails = dataStoresDetails.get(primaryStore.getUuid()); - - break; - } - } + if (details != null) { + managed = Boolean.parseBoolean(details.get(DiskTO.MANAGED)); } - if (rootDiskDataStoreDetails != null) { - return rootDiskDataStoreDetails.second(); + if (managed) { + String datastoreName = VmwareResource.getDatastoreName(details.get(DiskTO.IQN)); + rootDiskDataStoreDetails = dataStoresDetails.get(datastoreName); + } else { + DataStoreTO primaryStore = vol.getData().getDataStore(); + rootDiskDataStoreDetails = dataStoresDetails.get(primaryStore.getUuid()); } - return null; + return rootDiskDataStoreDetails; } private String getPvlanInfo(NicTO nicTo) { diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java index 35f77e2e358..5eb5c3e0d6a 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -35,6 +35,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import com.cloud.hypervisor.vmware.mo.VirtualStorageObjectManagerMO; +import com.cloud.hypervisor.vmware.mo.VirtualMachineDiskInfoBuilder; import com.vmware.vim25.BaseConfigInfoDiskFileBackingInfo; import com.vmware.vim25.VStorageObject; import com.vmware.vim25.VirtualDiskType; @@ -2081,6 +2082,7 @@ public class VmwareStorageProcessor implements StorageProcessor { DatastoreMO dsMo = new DatastoreMO(context, morDs); String datastoreVolumePath; + boolean datastoreChangeObserved = false; if (isAttach) { if (isManaged) { @@ -2106,6 +2108,25 @@ public class VmwareStorageProcessor implements StorageProcessor { if (isManaged) { datastoreVolumePath = dsMo.getDatastorePath((vmdkPath != null ? vmdkPath : dsMo.getName()) + ".vmdk"); } else { + String datastoreUUID = primaryStore.getUuid(); + if (disk.getDetails().get(DiskTO.PROTOCOL_TYPE) != null && disk.getDetails().get(DiskTO.PROTOCOL_TYPE).equalsIgnoreCase("DatastoreCluster")) { + DatastoreMO diskDatastoreMoFromVM = getDiskDatastoreMofromVM(hyperHost, context, vmMo, disk); + if (diskDatastoreMoFromVM != null) { + String actualPoolUuid = diskDatastoreMoFromVM.getCustomFieldValue(CustomFieldConstants.CLOUD_UUID); + if (!actualPoolUuid.equalsIgnoreCase(primaryStore.getUuid())) { + s_logger.warn(String.format("Volume %s found to be in a different storage pool %s", volumeTO.getPath(), actualPoolUuid)); + datastoreChangeObserved = true; + datastoreUUID = actualPoolUuid; + } + } + } + if (storagePort == DEFAULT_NFS_PORT) { + morDs = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, isManaged ? VmwareResource.getDatastoreName(diskUuid) : datastoreUUID); + } else { + morDs = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, isManaged ? VmwareResource.getDatastoreName(iScsiName) : datastoreUUID); + } + dsMo = new DatastoreMO(context, morDs); + datastoreVolumePath = VmwareStorageLayoutHelper.getLegacyDatastorePathFromVmdkFileName(dsMo, volumeTO.getPath() + ".vmdk"); if (!dsMo.fileExists(datastoreVolumePath)) { datastoreVolumePath = VmwareStorageLayoutHelper.getVmwareDatastorePathFromVmdkFileName(dsMo, vmName, volumeTO.getPath() + ".vmdk"); @@ -2147,6 +2168,8 @@ public class VmwareStorageProcessor implements StorageProcessor { VmwareStorageLayoutHelper.syncVolumeToRootFolder(dsMo.getOwnerDatacenter().first(), dsMo, volumeTO.getPath(), vmName, VmwareManager.s_vmwareSearchExcludeFolder.value()); } } + if (datastoreChangeObserved) + answer.setContextParam("datastoreName", dsMo.getName()); } return answer; @@ -2172,6 +2195,44 @@ public class VmwareStorageProcessor implements StorageProcessor { } } + private DatastoreMO getDiskDatastoreMofromVM(VmwareHypervisorHost hyperHost, VmwareContext context, + VirtualMachineMO vmMo, DiskTO disk) throws Exception { + assert (hyperHost != null) && (context != null); + List> diskDatastores = vmMo.getAllDiskDatastores(); + VirtualMachineDiskInfoBuilder diskInfoBuilder = vmMo.getDiskInfoBuilder(); + VolumeObjectTO volume = (VolumeObjectTO) disk.getData(); + String diskBackingFileBaseName = volume.getPath(); + for (Pair diskDatastore : diskDatastores) { + DatastoreMO dsMo = new DatastoreMO(hyperHost.getContext(), diskDatastore.second()); + String dsName = dsMo.getName(); + + VirtualMachineDiskInfo diskInfo = diskInfoBuilder.getDiskInfoByBackingFileBaseName(diskBackingFileBaseName, dsName); + if (diskInfo != null) { + s_logger.info("Found existing disk info from volume path: " + volume.getPath()); + return dsMo; + } else { + String chainInfo = volume.getChainInfo(); + if (chainInfo != null) { + VirtualMachineDiskInfo infoInChain = _gson.fromJson(chainInfo, VirtualMachineDiskInfo.class); + if (infoInChain != null) { + String[] disks = infoInChain.getDiskChain(); + if (disks.length > 0) { + for (String diskPath : disks) { + DatastoreFile file = new DatastoreFile(diskPath); + diskInfo = diskInfoBuilder.getDiskInfoByBackingFileBaseName(file.getFileBaseName(), dsName); + if (diskInfo != null) { + s_logger.info("Found existing disk from chain info: " + diskPath); + return dsMo; + } + } + } + } + } + } + } + return null; + } + private boolean expandVirtualDisk(VirtualMachineMO vmMo, String datastoreVolumePath, long currentSizeInBytes) throws Exception { long currentSizeInKB = currentSizeInBytes / 1024; diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index 4127b9f5fcc..3fdf534b7f4 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -2025,6 +2025,14 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic DataTO volTO = volFactory.getVolume(volume.getId()).getTO(); DiskTO disk = new DiskTO(volTO, volume.getDeviceId(), volume.getPath(), volume.getVolumeType()); + Map details = new HashMap(); + disk.setDetails(details); + if (volume.getPoolId() != null) { + StoragePoolVO poolVO = _storagePoolDao.findById(volume.getPoolId()); + if (poolVO.getParent() != 0L) { + details.put(DiskTO.PROTOCOL_TYPE, Storage.StoragePoolType.DatastoreCluster.toString()); + } + } DettachCommand cmd = new DettachCommand(disk, vm.getInstanceName()); @@ -2045,6 +2053,17 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic if (!sendCommand || (answer != null && answer.getResult())) { // Mark the volume as detached _volsDao.detachVolume(volume.getId()); + String datastoreName = answer.getContextParam("datastoreName"); + if (datastoreName != null) { + List storagePoolVOS = _storagePoolDao.listPoolsByLikePath(datastoreName); + if (storagePoolVOS != null && !storagePoolVOS.isEmpty()) { + VolumeVO volumeVO = _volsDao.findById(volumeId); + volumeVO.setPoolId(storagePoolVOS.get(0).getId()); + _volsDao.update(volumeVO.getId(), volumeVO); + } else { + s_logger.warn(String.format("Unable to find datastore %s while updating the new datastore of the volume %d", datastoreName, volumeId)); + } + } // volume.getPoolId() should be null if the VM we are detaching the disk from has never been started before if (volume.getPoolId() != null) { @@ -3041,6 +3060,14 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic details.put(DiskTO.CHAP_TARGET_USERNAME, chapInfo.getTargetUsername()); details.put(DiskTO.CHAP_TARGET_SECRET, chapInfo.getTargetSecret()); } + + if (volumeToAttach.getPoolId() != null) { + StoragePoolVO poolVO = _storagePoolDao.findById(volumeToAttach.getPoolId()); + if (poolVO.getParent() != 0L) { + details.put(DiskTO.PROTOCOL_TYPE, Storage.StoragePoolType.DatastoreCluster.toString()); + } + } + _userVmDao.loadDetails(vm); Map controllerInfo = new HashMap(); controllerInfo.put(VmDetailConstants.ROOT_DISK_CONTROLLER, vm.getDetail(VmDetailConstants.ROOT_DISK_CONTROLLER)); diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java index f99c8f70393..44eb47b93d1 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java @@ -2637,7 +2637,7 @@ public class VirtualMachineMO extends BaseMO { VirtualDeviceBackingInfo backingInfo = ((VirtualDisk)device).getBacking(); if (backingInfo instanceof VirtualDiskFlatVer2BackingInfo) { VirtualDiskFlatVer2BackingInfo diskBackingInfo = (VirtualDiskFlatVer2BackingInfo)backingInfo; - disks.add(new Pair(new Integer(device.getKey()), diskBackingInfo.getDatastore())); + disks.add(new Pair(new Integer(device.getUnitNumber()), diskBackingInfo.getDatastore())); } } } From 2f0387772368cd4ca06d38658e696aaf90c1cb2e Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Wed, 30 Sep 2020 15:16:21 +0530 Subject: [PATCH 113/164] Search in possible folders for volume while dettaching the volume --- .../com/cloud/storage/resource/VmwareStorageProcessor.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java index 5eb5c3e0d6a..fe58da56b1b 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -2131,7 +2131,10 @@ public class VmwareStorageProcessor implements StorageProcessor { if (!dsMo.fileExists(datastoreVolumePath)) { datastoreVolumePath = VmwareStorageLayoutHelper.getVmwareDatastorePathFromVmdkFileName(dsMo, vmName, volumeTO.getPath() + ".vmdk"); } - if (!dsMo.fileExists(datastoreVolumePath)) { + if (!dsMo.folderExists(String.format("[%s]", dsMo.getName()), vmName) || !dsMo.fileExists(datastoreVolumePath)) { + datastoreVolumePath = VmwareStorageLayoutHelper.getVmwareDatastorePathFromVmdkFileName(dsMo, volumeTO.getPath(), volumeTO.getPath() + ".vmdk"); + } + if (!dsMo.folderExists(String.format("[%s]", dsMo.getName()), volumeTO.getPath()) || !dsMo.fileExists(datastoreVolumePath)) { datastoreVolumePath = dsMo.searchFileInSubFolders(volumeTO.getPath() + ".vmdk", true, null); } } From 9b51a706db75eb2fa5f463c44018b0af6f3ebb08 Mon Sep 17 00:00:00 2001 From: nvazquez Date: Wed, 30 Sep 2020 10:25:57 -0300 Subject: [PATCH 114/164] Set deploy-as-is to default on VMware --- .../template/VirtualMachineTemplate.java | 2 - .../cloud/vm/VirtualMachineManagerImpl.java | 10 +- .../orchestration/VolumeOrchestrator.java | 6 +- .../java/com/cloud/storage/VMTemplateVO.java | 15 +-- .../META-INF/db/schema-41400to41500.sql | 112 ------------------ .../storage/image/TemplateServiceImpl.java | 6 +- .../storage/image/store/TemplateObject.java | 6 +- .../image/BaseImageStoreDriverImpl.java | 3 +- .../storage/volume/VolumeObject.java | 4 +- .../storage/volume/VolumeServiceImpl.java | 4 +- .../com/cloud/hypervisor/guru/VMwareGuru.java | 2 +- .../hypervisor/guru/VmwareVmImplementer.java | 2 +- .../deploy/DeploymentPlanningManagerImpl.java | 2 +- .../storage/ImageStoreUploadMonitorImpl.java | 2 +- .../com/cloud/storage/TemplateProfile.java | 8 +- .../template/HypervisorTemplateAdapter.java | 6 +- .../cloud/template/TemplateAdapterBase.java | 4 +- .../cloud/template/TemplateManagerImpl.java | 2 +- .../vm/DeploymentPlanningManagerImplTest.java | 5 - 19 files changed, 28 insertions(+), 173 deletions(-) diff --git a/api/src/main/java/com/cloud/template/VirtualMachineTemplate.java b/api/src/main/java/com/cloud/template/VirtualMachineTemplate.java index 95d1ebf0b87..5177e51d401 100644 --- a/api/src/main/java/com/cloud/template/VirtualMachineTemplate.java +++ b/api/src/main/java/com/cloud/template/VirtualMachineTemplate.java @@ -138,6 +138,4 @@ public interface VirtualMachineTemplate extends ControlledEntity, Identity, Inte void incrUpdatedCount(); Date getUpdated(); - - boolean isDeployAsIs(); } diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index e8a96f1d6fb..2cce812df41 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -597,12 +597,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac //remove the overcommit details from the uservm details userVmDetailsDao.removeDetails(vm.getId()); - // Remove details if VM deploy as-is - long templateId = vm.getTemplateId(); - VMTemplateVO template = _templateDao.findById(templateId); - if (template != null && template.isDeployAsIs()) { - userVmDeployAsIsDetailsDao.removeDetails(vm.getId()); - } + // Remove deploy as-is (if any) + userVmDeployAsIsDetailsDao.removeDetails(vm.getId()); // send hypervisor-dependent commands before removing final List finalizeExpungeCommands = hvGuru.finalizeExpunge(vm); @@ -1116,7 +1112,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac if (dest != null) { avoids.addHost(dest.getHost().getId()); - if (!template.isDeployAsIs()) { + if (template.getHypervisorType() == HypervisorType.VMware && vm.getType() == VirtualMachine.Type.User) { journal.record("Deployment found ", vmProfile, dest); } } diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java index b4d662d449c..aedaf4018f3 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java @@ -759,7 +759,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati Long size = _tmpltMgr.getTemplateSize(template.getId(), vm.getDataCenterId()); if (rootDisksize != null) { - if (template.isDeployAsIs()) { + if (vm.getHypervisorType() == HypervisorType.VMware && vm.getType() == VirtualMachine.Type.User) { // Volume size specified from template deploy-as-is size = rootDisksize; } else { @@ -824,7 +824,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati int volumesNumber = 1; List templateAsIsDisks = null; String configurationId = null; - if (template.isDeployAsIs()) { + if (vm.getHypervisorType() == HypervisorType.VMware && vm.getType() == VirtualMachine.Type.User) { UserVmDetailVO configurationDetail = userVmDetailsDao.findDetail(vm.getId(), VmDetailConstants.DEPLOY_AS_IS_CONFIGURATION); if (configurationDetail != null) { configurationId = configurationDetail.getValue(); @@ -849,7 +849,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati String volumeName = name; Long volumeSize = rootDisksize; long deviceId = type.equals(Type.ROOT) ? 0L : 1L; - if (template.isDeployAsIs()) { + if (vm.getHypervisorType() == HypervisorType.VMware && vm.getType() == VirtualMachine.Type.User) { int volumeNameSuffix = templateAsIsDisks.get(number).getDiskNumber(); volumeName = String.format("%s-%d", volumeName, volumeNameSuffix); volumeSize = templateAsIsDisks.get(number).getVirtualSize(); diff --git a/engine/schema/src/main/java/com/cloud/storage/VMTemplateVO.java b/engine/schema/src/main/java/com/cloud/storage/VMTemplateVO.java index 61df40e50d8..3bbebf9ba5f 100644 --- a/engine/schema/src/main/java/com/cloud/storage/VMTemplateVO.java +++ b/engine/schema/src/main/java/com/cloud/storage/VMTemplateVO.java @@ -152,9 +152,6 @@ public class VMTemplateVO implements VirtualMachineTemplate { @Column(name = "parent_template_id") private Long parentTemplateId; - @Column(name = "deploy_as_is") - private boolean deployAsIs; - @Override public String getUniqueName() { return uniqueName; @@ -196,8 +193,7 @@ public class VMTemplateVO implements VirtualMachineTemplate { } public VMTemplateVO(long id, String name, ImageFormat format, boolean isPublic, boolean featured, boolean isExtractable, TemplateType type, String url, boolean requiresHvm, int bits, long accountId, String cksum, String displayText, boolean enablePassword, long guestOSId, boolean bootable, - HypervisorType hyperType, String templateTag, Map details, boolean sshKeyEnabled, boolean isDynamicallyScalable, boolean directDownload, - boolean deployAsIs) { + HypervisorType hyperType, String templateTag, Map details, boolean sshKeyEnabled, boolean isDynamicallyScalable, boolean directDownload) { this(id, name, format, @@ -222,7 +218,6 @@ public class VMTemplateVO implements VirtualMachineTemplate { dynamicallyScalable = isDynamicallyScalable; state = State.Active; this.directDownload = directDownload; - this.deployAsIs = deployAsIs; } public static VMTemplateVO createPreHostIso(Long id, String uniqueName, String name, ImageFormat format, boolean isPublic, boolean featured, TemplateType type, @@ -640,12 +635,4 @@ public class VMTemplateVO implements VirtualMachineTemplate { public void setParentTemplateId(Long parentTemplateId) { this.parentTemplateId = parentTemplateId; } - - @Override public boolean isDeployAsIs() { - return deployAsIs; - } - - public void setDeployAsIs(boolean deployAsIs) { - this.deployAsIs = deployAsIs; - } } diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql b/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql index 76907037af1..01969d6e215 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql @@ -260,118 +260,6 @@ CREATE VIEW `cloud`.`storage_pool_view` AS LEFT JOIN `async_job` ON (((`async_job`.`instance_id` = `storage_pool`.`id`) AND (`async_job`.`instance_type` = 'StoragePool') AND (`async_job`.`job_status` = 0)))); --- Add passthrough instruction for appliance deployments -ALTER TABLE `cloud`.`vm_template` ADD COLUMN `deploy_as_is` tinyint(1) NOT NULL DEFAULT '0' COMMENT 'True if the template should be deployed with disks and networks as defined by OVF'; - --- Extend the template details value field -ALTER TABLE `cloud`.`vm_template_details` MODIFY COLUMN `value` TEXT CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL; - --- Changes to template_view for both deploying multidisk OVA/vApp as is -DROP VIEW IF EXISTS `cloud`.`template_view`; -CREATE VIEW `cloud`.`template_view` AS - SELECT - `vm_template`.`id` AS `id`, - `vm_template`.`uuid` AS `uuid`, - `vm_template`.`unique_name` AS `unique_name`, - `vm_template`.`name` AS `name`, - `vm_template`.`public` AS `public`, - `vm_template`.`featured` AS `featured`, - `vm_template`.`type` AS `type`, - `vm_template`.`hvm` AS `hvm`, - `vm_template`.`bits` AS `bits`, - `vm_template`.`url` AS `url`, - `vm_template`.`format` AS `format`, - `vm_template`.`created` AS `created`, - `vm_template`.`checksum` AS `checksum`, - `vm_template`.`display_text` AS `display_text`, - `vm_template`.`enable_password` AS `enable_password`, - `vm_template`.`dynamically_scalable` AS `dynamically_scalable`, - `vm_template`.`state` AS `template_state`, - `vm_template`.`guest_os_id` AS `guest_os_id`, - `guest_os`.`uuid` AS `guest_os_uuid`, - `guest_os`.`display_name` AS `guest_os_name`, - `vm_template`.`bootable` AS `bootable`, - `vm_template`.`prepopulate` AS `prepopulate`, - `vm_template`.`cross_zones` AS `cross_zones`, - `vm_template`.`hypervisor_type` AS `hypervisor_type`, - `vm_template`.`extractable` AS `extractable`, - `vm_template`.`template_tag` AS `template_tag`, - `vm_template`.`sort_key` AS `sort_key`, - `vm_template`.`removed` AS `removed`, - `vm_template`.`enable_sshkey` AS `enable_sshkey`, - `parent_template`.`id` AS `parent_template_id`, - `parent_template`.`uuid` AS `parent_template_uuid`, - `source_template`.`id` AS `source_template_id`, - `source_template`.`uuid` AS `source_template_uuid`, - `account`.`id` AS `account_id`, - `account`.`uuid` AS `account_uuid`, - `account`.`account_name` AS `account_name`, - `account`.`type` AS `account_type`, - `domain`.`id` AS `domain_id`, - `domain`.`uuid` AS `domain_uuid`, - `domain`.`name` AS `domain_name`, - `domain`.`path` AS `domain_path`, - `projects`.`id` AS `project_id`, - `projects`.`uuid` AS `project_uuid`, - `projects`.`name` AS `project_name`, - `data_center`.`id` AS `data_center_id`, - `data_center`.`uuid` AS `data_center_uuid`, - `data_center`.`name` AS `data_center_name`, - `launch_permission`.`account_id` AS `lp_account_id`, - `template_store_ref`.`store_id` AS `store_id`, - `image_store`.`scope` AS `store_scope`, - `template_store_ref`.`state` AS `state`, - `template_store_ref`.`download_state` AS `download_state`, - `template_store_ref`.`download_pct` AS `download_pct`, - `template_store_ref`.`error_str` AS `error_str`, - `template_store_ref`.`size` AS `size`, - `template_store_ref`.physical_size AS `physical_size`, - `template_store_ref`.`destroyed` AS `destroyed`, - `template_store_ref`.`created` AS `created_on_store`, - `vm_template_details`.`name` AS `detail_name`, - `vm_template_details`.`value` AS `detail_value`, - `resource_tags`.`id` AS `tag_id`, - `resource_tags`.`uuid` AS `tag_uuid`, - `resource_tags`.`key` AS `tag_key`, - `resource_tags`.`value` AS `tag_value`, - `resource_tags`.`domain_id` AS `tag_domain_id`, - `domain`.`uuid` AS `tag_domain_uuid`, - `domain`.`name` AS `tag_domain_name`, - `resource_tags`.`account_id` AS `tag_account_id`, - `account`.`account_name` AS `tag_account_name`, - `resource_tags`.`resource_id` AS `tag_resource_id`, - `resource_tags`.`resource_uuid` AS `tag_resource_uuid`, - `resource_tags`.`resource_type` AS `tag_resource_type`, - `resource_tags`.`customer` AS `tag_customer`, - CONCAT(`vm_template`.`id`, - '_', - IFNULL(`data_center`.`id`, 0)) AS `temp_zone_pair`, - `vm_template`.`direct_download` AS `direct_download`, - `vm_template`.`deploy_as_is` AS `deploy_as_is` - FROM - (((((((((((((`vm_template` - JOIN `guest_os` ON ((`guest_os`.`id` = `vm_template`.`guest_os_id`))) - JOIN `account` ON ((`account`.`id` = `vm_template`.`account_id`))) - JOIN `domain` ON ((`domain`.`id` = `account`.`domain_id`))) - LEFT JOIN `projects` ON ((`projects`.`project_account_id` = `account`.`id`))) - LEFT JOIN `vm_template_details` ON ((`vm_template_details`.`template_id` = `vm_template`.`id`))) - LEFT JOIN `vm_template` `source_template` ON ((`source_template`.`id` = `vm_template`.`source_template_id`))) - LEFT JOIN `template_store_ref` ON (((`template_store_ref`.`template_id` = `vm_template`.`id`) - AND (`template_store_ref`.`store_role` = 'Image') - AND (`template_store_ref`.`destroyed` = 0)))) - LEFT JOIN `vm_template` `parent_template` ON ((`parent_template`.`id` = `vm_template`.`parent_template_id`))) - LEFT JOIN `image_store` ON ((ISNULL(`image_store`.`removed`) - AND (`template_store_ref`.`store_id` IS NOT NULL) - AND (`image_store`.`id` = `template_store_ref`.`store_id`)))) - LEFT JOIN `template_zone_ref` ON (((`template_zone_ref`.`template_id` = `vm_template`.`id`) - AND ISNULL(`template_store_ref`.`store_id`) - AND ISNULL(`template_zone_ref`.`removed`)))) - LEFT JOIN `data_center` ON (((`image_store`.`data_center_id` = `data_center`.`id`) - OR (`template_zone_ref`.`zone_id` = `data_center`.`id`)))) - LEFT JOIN `launch_permission` ON ((`launch_permission`.`template_id` = `vm_template`.`id`))) - LEFT JOIN `resource_tags` ON (((`resource_tags`.`resource_id` = `vm_template`.`id`) - AND ((`resource_tags`.`resource_type` = 'Template') - OR (`resource_tags`.`resource_type` = 'ISO'))))); -- Add mincpu, maxcpu, minmemory and maxmemory to the view supporting constrained offerings DROP VIEW IF EXISTS `cloud`.`service_offering_view`; diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java index ed9359d952a..a10aa203ebb 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java @@ -412,7 +412,7 @@ public class TemplateServiceImpl implements TemplateService { VirtualMachineTemplate.Event event = VirtualMachineTemplate.Event.OperationSucceeded; // For multi-disk OVA, check and create data disk templates if (tmplt.getFormat().equals(ImageFormat.OVA)) { - if (!createOvaDataDiskTemplates(_templateFactory.getTemplate(tmlpt.getId(), store), tmplt.isDeployAsIs())) { + if (!createOvaDataDiskTemplates(_templateFactory.getTemplate(tmlpt.getId(), store), true)) { event = VirtualMachineTemplate.Event.OperationFailed; } } @@ -710,7 +710,7 @@ public class TemplateServiceImpl implements TemplateService { // For multi-disk OVA, check and create data disk templates if (template.getFormat().equals(ImageFormat.OVA)) { - if (!createOvaDataDiskTemplates(template, template.isDeployAsIs())) { + if (!createOvaDataDiskTemplates(template, true)) { template.processEvent(ObjectInDataStoreStateMachine.Event.OperationFailed); result.setResult(callbackResult.getResult()); if (parentCallback != null) { @@ -799,7 +799,7 @@ public class TemplateServiceImpl implements TemplateService { String templateName = dataDiskTemplate.isIso() ? dataDiskTemplate.getPath().substring(dataDiskTemplate.getPath().lastIndexOf(File.separator) + 1) : template.getName() + suffix + diskCount; VMTemplateVO templateVO = new VMTemplateVO(templateId, templateName, format, false, false, false, ttype, template.getUrl(), template.requiresHvm(), template.getBits(), template.getAccountId(), null, templateName, false, guestOsId, false, template.getHypervisorType(), null, - null, false, false, false, false); + null, false, false, false); if (dataDiskTemplate.isIso()){ templateVO.setUniqueName(templateName); } diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/TemplateObject.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/TemplateObject.java index b7a44cd4f08..6d7ebebdc1d 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/TemplateObject.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/TemplateObject.java @@ -376,10 +376,10 @@ public class TemplateObject implements TemplateInfo { @Override public boolean isDeployAsIs() { - if (this.imageVO == null) { - return false; + if (getHypervisorType() == HypervisorType.VMware) { + return true; } - return this.imageVO.isDeployAsIs(); + return false; } public void setInstallPath(String installPath) { diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java index 37a7985ce6f..340347fc6f6 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java @@ -32,6 +32,7 @@ import java.util.stream.Collectors; import javax.inject.Inject; +import com.cloud.hypervisor.Hypervisor; import com.cloud.storage.Upload; import org.apache.cloudstack.storage.image.deployasis.DeployAsIsHelper; import org.apache.log4j.Logger; @@ -203,7 +204,7 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver { TemplateDataStoreVO tmpltStoreVO = _templateStoreDao.findByStoreTemplate(store.getId(), obj.getId()); if (tmpltStoreVO != null) { if (tmpltStoreVO.getDownloadState() == VMTemplateStorageResourceAssoc.Status.DOWNLOADED) { - if (template.isDeployAsIs()) { + if (template.getHypervisorType() == Hypervisor.HypervisorType.VMware) { boolean persistDeployAsIs = deployAsIsHelper.persistTemplateDeployAsIsDetails(template.getId(), answer, tmpltStoreVO); if (!persistDeployAsIs) { LOGGER.info("Failed persisting deploy-as-is template details for template " + template.getName()); diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java index 45509c51c43..d40b30903ee 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java @@ -21,7 +21,6 @@ import java.util.Date; import javax.inject.Inject; import com.cloud.storage.MigrationOptions; -import com.cloud.storage.VMTemplateVO; import com.cloud.storage.VolumeDetailVO; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VolumeDetailsDao; @@ -446,8 +445,7 @@ public class VolumeObject implements VolumeInfo { @Override public boolean isDeployAsIs() { - VMTemplateVO template = templateDao.findById(getTemplateId()); - return template != null && template.isDeployAsIs(); + return getHypervisorType() == HypervisorType.VMware; } @Override diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java index 3ccb7be8f60..53a36ede6bd 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java @@ -575,7 +575,7 @@ public class VolumeServiceImpl implements VolumeService { s_logger.info("Unable to acquire lock on VMTemplateStoragePool " + templatePoolRefId); } templatePoolRef = _tmpltPoolDao.findByPoolTemplate(dataStore.getId(), template.getId(), deployAsIsConfiguration); - if (templatePoolRef != null && templatePoolRef.getState() == ObjectInDataStoreStateMachine.State.Ready && !template.isDeployAsIs()) { + if (templatePoolRef != null && templatePoolRef.getState() == ObjectInDataStoreStateMachine.State.Ready) { s_logger.info( "Unable to acquire lock on VMTemplateStoragePool " + templatePoolRefId + ", But Template " + template.getUniqueName() + " is already copied to primary storage, skip copying"); createVolumeFromBaseImageAsync(volume, templateOnPrimaryStoreObj, dataStore, future); @@ -588,7 +588,7 @@ public class VolumeServiceImpl implements VolumeService { s_logger.info("lock is acquired for VMTemplateStoragePool " + templatePoolRefId); } try { - if (templatePoolRef.getState() == ObjectInDataStoreStateMachine.State.Ready && !template.isDeployAsIs()) { + if (templatePoolRef.getState() == ObjectInDataStoreStateMachine.State.Ready) { s_logger.info("Template " + template.getUniqueName() + " is already copied to primary storage, skip copying"); createVolumeFromBaseImageAsync(volume, templateOnPrimaryStoreObj, dataStore, future); return; diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java index d48a5d9b101..6e301985f83 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java @@ -598,7 +598,7 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co private VMTemplateVO createVMTemplateRecord(String vmInternalName, long guestOsId, long accountId) { Long nextTemplateId = vmTemplateDao.getNextInSequence(Long.class, "id"); VMTemplateVO templateVO = new VMTemplateVO(nextTemplateId, "Imported-from-" + vmInternalName, Storage.ImageFormat.OVA, false, false, false, Storage.TemplateType.USER, null, - false, 64, accountId, null, "Template imported from VM " + vmInternalName, false, guestOsId, false, HypervisorType.VMware, null, null, false, false, false, false); + false, 64, accountId, null, "Template imported from VM " + vmInternalName, false, guestOsId, false, HypervisorType.VMware, null, null, false, false, false); return vmTemplateDao.persist(templateVO); } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java index c12c5c20e2d..6638d769df8 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java @@ -105,7 +105,7 @@ class VmwareVmImplementer { VirtualMachineTO implement(VirtualMachineProfile vm, VirtualMachineTO to, long clusterId) { to.setBootloader(VirtualMachineTemplate.BootloaderType.HVM); - boolean deployAsIs = vm.getTemplate().isDeployAsIs(); + boolean deployAsIs = vm.getType() == VirtualMachine.Type.User; HostVO host = hostDao.findById(vm.getVirtualMachine().getHostId()); Map details = to.getDetails(); if (details == null) diff --git a/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java b/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java index eb99a8eefe3..51d589fcd81 100644 --- a/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java +++ b/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java @@ -561,7 +561,7 @@ StateListener { * Display storage in the logs by default if the template is not deploy-as-is. */ private boolean getDisplayStorageFromVmProfile(VirtualMachineProfile vmProfile) { - return vmProfile == null || vmProfile.getTemplate() == null || !vmProfile.getTemplate().isDeployAsIs(); + return vmProfile.getHypervisorType() != HypervisorType.VMware || vmProfile.getType() != VirtualMachine.Type.User; } @Override diff --git a/server/src/main/java/com/cloud/storage/ImageStoreUploadMonitorImpl.java b/server/src/main/java/com/cloud/storage/ImageStoreUploadMonitorImpl.java index 1ce5b362eb9..5174b2613ab 100755 --- a/server/src/main/java/com/cloud/storage/ImageStoreUploadMonitorImpl.java +++ b/server/src/main/java/com/cloud/storage/ImageStoreUploadMonitorImpl.java @@ -412,7 +412,7 @@ public class ImageStoreUploadMonitorImpl extends ManagerBase implements ImageSto if (tmpTemplate.getFormat().equals(Storage.ImageFormat.OVA)) { final DataStore store = dataStoreManager.getDataStore(templateDataStore.getDataStoreId(), templateDataStore.getDataStoreRole()); final TemplateInfo templateInfo = templateFactory.getTemplate(tmpTemplate.getId(), store); - if (!templateService.createOvaDataDiskTemplates(templateInfo, template.isDeployAsIs())) { + if (!templateService.createOvaDataDiskTemplates(templateInfo, true)) { tmpTemplateDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.ABANDONED); tmpTemplateDataStore.setState(State.Failed); stateMachine.transitTo(tmpTemplate, VirtualMachineTemplate.Event.OperationFailed, null, _templateDao); diff --git a/server/src/main/java/com/cloud/storage/TemplateProfile.java b/server/src/main/java/com/cloud/storage/TemplateProfile.java index b90409480bc..304b652a589 100644 --- a/server/src/main/java/com/cloud/storage/TemplateProfile.java +++ b/server/src/main/java/com/cloud/storage/TemplateProfile.java @@ -52,7 +52,6 @@ public class TemplateProfile { Boolean isDynamicallyScalable; TemplateType templateType; Boolean directDownload; - Boolean deployAsIs; Long size; public TemplateProfile(Long templateId, Long userId, String name, String displayText, Integer bits, Boolean passwordEnabled, Boolean requiresHvm, String url, @@ -96,7 +95,7 @@ public class TemplateProfile { Boolean isPublic, Boolean featured, Boolean isExtractable, ImageFormat format, Long guestOsId, List zoneId, HypervisorType hypervisorType, String accountName, Long domainId, Long accountId, String chksum, Boolean bootable, String templateTag, Map details, - Boolean sshKeyEnabled, Long imageStoreId, Boolean isDynamicallyScalable, TemplateType templateType, Boolean directDownload, Boolean deployAsIs) { + Boolean sshKeyEnabled, Long imageStoreId, Boolean isDynamicallyScalable, TemplateType templateType, Boolean directDownload) { this(templateId, userId, name, @@ -123,7 +122,6 @@ public class TemplateProfile { this.isDynamicallyScalable = isDynamicallyScalable; this.templateType = templateType; this.directDownload = directDownload; - this.deployAsIs = deployAsIs; } public Long getTemplateId() { @@ -333,8 +331,4 @@ public class TemplateProfile { public void setSize(Long size) { this.size = size; } - - public boolean isDeployAsIs() { - return this.deployAsIs; - } } diff --git a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java index 18d37d0b8ad..c080ffd2f25 100644 --- a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java +++ b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java @@ -601,10 +601,8 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { // Remove template details templateDetailsDao.removeDetails(template.getId()); - // Remove deploy-as-is details - if (template.isDeployAsIs()) { - templateDeployAsIsDetailsDao.removeDetails(template.getId()); - } + // Remove deploy-as-is details (if any) + templateDeployAsIsDetailsDao.removeDetails(template.getId()); } return success; diff --git a/server/src/main/java/com/cloud/template/TemplateAdapterBase.java b/server/src/main/java/com/cloud/template/TemplateAdapterBase.java index 45a10a8c4b3..9cd2b6f791c 100644 --- a/server/src/main/java/com/cloud/template/TemplateAdapterBase.java +++ b/server/src/main/java/com/cloud/template/TemplateAdapterBase.java @@ -265,7 +265,7 @@ public abstract class TemplateAdapterBase extends AdapterBase implements Templat CallContext.current().setEventDetails("Id: " + id + " name: " + name); return new TemplateProfile(id, userId, name, displayText, bits, passwordEnabled, requiresHVM, url, isPublic, featured, isExtractable, imgfmt, guestOSId, zoneIdList, hypervisorType, templateOwner.getAccountName(), templateOwner.getDomainId(), templateOwner.getAccountId(), chksum, bootable, templateTag, details, - sshkeyEnabled, null, isDynamicallyScalable, templateType, directDownload, deployAsIs); + sshkeyEnabled, null, isDynamicallyScalable, templateType, directDownload); } @@ -376,7 +376,7 @@ public abstract class TemplateAdapterBase extends AdapterBase implements Templat new VMTemplateVO(profile.getTemplateId(), profile.getName(), profile.getFormat(), profile.isPublic(), profile.isFeatured(), profile.isExtractable(), profile.getTemplateType(), profile.getUrl(), profile.isRequiresHVM(), profile.getBits(), profile.getAccountId(), profile.getCheckSum(), profile.getDisplayText(), profile.isPasswordEnabled(), profile.getGuestOsId(), profile.isBootable(), profile.getHypervisorType(), - profile.getTemplateTag(), profile.getDetails(), profile.isSshKeyEnabled(), profile.IsDynamicallyScalable(), profile.isDirectDownload(), profile.isDeployAsIs()); + profile.getTemplateTag(), profile.getDetails(), profile.isSshKeyEnabled(), profile.IsDynamicallyScalable(), profile.isDirectDownload()); template.setState(initialState); if (profile.isDirectDownload()) { diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index c58b2f1ad9a..519072272dd 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -1898,7 +1898,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, } privateTemplate = new VMTemplateVO(nextTemplateId, name, ImageFormat.RAW, isPublic, featured, isExtractable, TemplateType.USER, null, requiresHvmValue, bitsValue, templateOwner.getId(), null, description, - passwordEnabledValue, guestOS.getId(), true, hyperType, templateTag, cmd.getDetails(), sshKeyEnabledValue, isDynamicScalingEnabled, false, false); + passwordEnabledValue, guestOS.getId(), true, hyperType, templateTag, cmd.getDetails(), sshKeyEnabledValue, isDynamicScalingEnabled, false); if (sourceTemplateId != null) { if (s_logger.isDebugEnabled()) { diff --git a/server/src/test/java/com/cloud/vm/DeploymentPlanningManagerImplTest.java b/server/src/test/java/com/cloud/vm/DeploymentPlanningManagerImplTest.java index d356570b633..71a6935a7d8 100644 --- a/server/src/test/java/com/cloud/vm/DeploymentPlanningManagerImplTest.java +++ b/server/src/test/java/com/cloud/vm/DeploymentPlanningManagerImplTest.java @@ -29,7 +29,6 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; import com.cloud.host.Host; -import com.cloud.storage.VMTemplateVO; import com.cloud.storage.dao.VMTemplateDao; import org.apache.cloudstack.affinity.dao.AffinityGroupDomainMapDao; import org.junit.Before; @@ -167,10 +166,6 @@ public class DeploymentPlanningManagerImplTest { Mockito.when(_plannerHostReserveDao.findById(Matchers.anyLong())).thenReturn(reservationVO); Mockito.when(_affinityGroupVMMapDao.countAffinityGroupsForVm(Matchers.anyLong())).thenReturn(0L); - VMTemplateVO template = Mockito.mock(VMTemplateVO.class); - Mockito.when(template.isDeployAsIs()).thenReturn(false); - Mockito.when(templateDao.findById(Mockito.anyLong())).thenReturn(template); - VMInstanceVO vm = new VMInstanceVO(); Mockito.when(vmProfile.getVirtualMachine()).thenReturn(vm); From 46d412d998ee3331633242e2d2271995527a75a3 Mon Sep 17 00:00:00 2001 From: nvazquez Date: Wed, 30 Sep 2020 11:33:27 -0300 Subject: [PATCH 115/164] Fix for system VMs --- .../cloudstack/engine/orchestration/VolumeOrchestrator.java | 2 +- .../java/com/cloud/hypervisor/guru/VmwareVmImplementer.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java index aedaf4018f3..de5e260a6e8 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java @@ -759,7 +759,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati Long size = _tmpltMgr.getTemplateSize(template.getId(), vm.getDataCenterId()); if (rootDisksize != null) { - if (vm.getHypervisorType() == HypervisorType.VMware && vm.getType() == VirtualMachine.Type.User) { + if (vm.getHypervisorType() == HypervisorType.VMware) { // Volume size specified from template deploy-as-is size = rootDisksize; } else { diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java index 6638d769df8..0e0dd8c56b1 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java @@ -105,7 +105,7 @@ class VmwareVmImplementer { VirtualMachineTO implement(VirtualMachineProfile vm, VirtualMachineTO to, long clusterId) { to.setBootloader(VirtualMachineTemplate.BootloaderType.HVM); - boolean deployAsIs = vm.getType() == VirtualMachine.Type.User; + boolean deployAsIs = true; HostVO host = hostDao.findById(vm.getVirtualMachine().getHostId()); Map details = to.getDetails(); if (details == null) From 08c0b07b59f9c765dacf67d89babb84d8ea14e97 Mon Sep 17 00:00:00 2001 From: nvazquez Date: Wed, 30 Sep 2020 23:34:41 -0300 Subject: [PATCH 116/164] Remove deployasis parameter from register template API --- .../command/user/template/RegisterTemplateCmd.java | 13 +------------ .../cloud/api/query/dao/TemplateJoinDaoImpl.java | 4 ++-- .../java/com/cloud/api/query/vo/TemplateJoinVO.java | 7 ------- 3 files changed, 3 insertions(+), 21 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java index 6decb945496..900c02e7c01 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java @@ -162,13 +162,6 @@ public class RegisterTemplateCmd extends BaseCmd implements UserCmd { description = "true if template should bypass Secondary Storage and be downloaded to Primary Storage on deployment") private Boolean directDownload; - @Parameter(name= ApiConstants.DEPLOY_AS_IS, - type = CommandType.BOOLEAN, - description = "VMware only: true if template should not strip and define disks and networks but leave those to the template definition", - since = "4.15" - ) - private Boolean deployAsIs; - ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -282,7 +275,7 @@ public class RegisterTemplateCmd extends BaseCmd implements UserCmd { } public Boolean isDeployAsIs() { - return deployAsIs == null ? false : deployAsIs; + return hypervisor != null && hypervisor.equalsIgnoreCase(Hypervisor.HypervisorType.VMware.toString()); } ///////////////////////////////////////////////////// @@ -347,9 +340,5 @@ public class RegisterTemplateCmd extends BaseCmd implements UserCmd { throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Parameter directdownload is only allowed for KVM templates"); } - - if (isDeployAsIs() && !getHypervisor().equalsIgnoreCase(Hypervisor.HypervisorType.VMware.toString())) { - throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Parameter deployasis is only allowed for VMware templates"); - } } } diff --git a/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDaoImpl.java index 92072c7a3c5..a8ae39fc6eb 100644 --- a/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDaoImpl.java @@ -253,7 +253,7 @@ public class TemplateJoinDaoImpl extends GenericDaoBaseWithTagInformation deployAsIsDetails = templateDeployAsIsDetailsDao.listDetails(template.getId()); for (TemplateDeployAsIsDetailVO deployAsIsDetailVO : deployAsIsDetails) { if (deployAsIsDetailVO.getName().startsWith(DeployAsIsConstants.HARDWARE_ITEM_PREFIX)) { diff --git a/server/src/main/java/com/cloud/api/query/vo/TemplateJoinVO.java b/server/src/main/java/com/cloud/api/query/vo/TemplateJoinVO.java index 91bb76336cc..25e3b0b5ff5 100644 --- a/server/src/main/java/com/cloud/api/query/vo/TemplateJoinVO.java +++ b/server/src/main/java/com/cloud/api/query/vo/TemplateJoinVO.java @@ -231,9 +231,6 @@ public class TemplateJoinVO extends BaseViewWithTagInformationVO implements Cont @Column(name = "direct_download") private boolean directDownload; - @Column(name = "deploy_as_is") - private boolean deployAsIs; - public TemplateJoinVO() { } @@ -493,10 +490,6 @@ public class TemplateJoinVO extends BaseViewWithTagInformationVO implements Cont return directDownload; } - public boolean isDeployAsIs() { - return deployAsIs; - } - public Object getParentTemplateId() { return parentTemplateId; } From 94bebe8792ca061ddda7794f9cb28d029d1e77d2 Mon Sep 17 00:00:00 2001 From: nvazquez Date: Thu, 1 Oct 2020 10:37:40 -0300 Subject: [PATCH 117/164] Revert back deploy as is column on templates but keep it as default for new templates --- .../template/VirtualMachineTemplate.java | 2 + .../cloud/vm/VirtualMachineManagerImpl.java | 2 +- .../orchestration/VolumeOrchestrator.java | 6 +- .../java/com/cloud/storage/VMTemplateVO.java | 15 ++- .../META-INF/db/schema-41400to41500.sql | 110 ++++++++++++++++++ .../storage/image/TemplateServiceImpl.java | 6 +- .../storage/image/store/TemplateObject.java | 6 +- .../image/BaseImageStoreDriverImpl.java | 3 +- .../storage/volume/VolumeObject.java | 4 +- .../com/cloud/hypervisor/guru/VMwareGuru.java | 2 +- .../hypervisor/guru/VmwareVmImplementer.java | 2 +- .../api/query/dao/TemplateJoinDaoImpl.java | 4 +- .../cloud/api/query/vo/TemplateJoinVO.java | 7 ++ .../deploy/DeploymentPlanningManagerImpl.java | 2 +- .../storage/ImageStoreUploadMonitorImpl.java | 2 +- .../com/cloud/storage/TemplateProfile.java | 8 +- .../cloud/template/TemplateAdapterBase.java | 4 +- .../cloud/template/TemplateManagerImpl.java | 2 +- .../vm/DeploymentPlanningManagerImplTest.java | 5 + 19 files changed, 168 insertions(+), 24 deletions(-) diff --git a/api/src/main/java/com/cloud/template/VirtualMachineTemplate.java b/api/src/main/java/com/cloud/template/VirtualMachineTemplate.java index 5177e51d401..95d1ebf0b87 100644 --- a/api/src/main/java/com/cloud/template/VirtualMachineTemplate.java +++ b/api/src/main/java/com/cloud/template/VirtualMachineTemplate.java @@ -138,4 +138,6 @@ public interface VirtualMachineTemplate extends ControlledEntity, Identity, Inte void incrUpdatedCount(); Date getUpdated(); + + boolean isDeployAsIs(); } diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 2cce812df41..8e9ec450b1e 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -1112,7 +1112,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac if (dest != null) { avoids.addHost(dest.getHost().getId()); - if (template.getHypervisorType() == HypervisorType.VMware && vm.getType() == VirtualMachine.Type.User) { + if (!template.isDeployAsIs()) { journal.record("Deployment found ", vmProfile, dest); } } diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java index de5e260a6e8..b4d662d449c 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java @@ -759,7 +759,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati Long size = _tmpltMgr.getTemplateSize(template.getId(), vm.getDataCenterId()); if (rootDisksize != null) { - if (vm.getHypervisorType() == HypervisorType.VMware) { + if (template.isDeployAsIs()) { // Volume size specified from template deploy-as-is size = rootDisksize; } else { @@ -824,7 +824,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati int volumesNumber = 1; List templateAsIsDisks = null; String configurationId = null; - if (vm.getHypervisorType() == HypervisorType.VMware && vm.getType() == VirtualMachine.Type.User) { + if (template.isDeployAsIs()) { UserVmDetailVO configurationDetail = userVmDetailsDao.findDetail(vm.getId(), VmDetailConstants.DEPLOY_AS_IS_CONFIGURATION); if (configurationDetail != null) { configurationId = configurationDetail.getValue(); @@ -849,7 +849,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati String volumeName = name; Long volumeSize = rootDisksize; long deviceId = type.equals(Type.ROOT) ? 0L : 1L; - if (vm.getHypervisorType() == HypervisorType.VMware && vm.getType() == VirtualMachine.Type.User) { + if (template.isDeployAsIs()) { int volumeNameSuffix = templateAsIsDisks.get(number).getDiskNumber(); volumeName = String.format("%s-%d", volumeName, volumeNameSuffix); volumeSize = templateAsIsDisks.get(number).getVirtualSize(); diff --git a/engine/schema/src/main/java/com/cloud/storage/VMTemplateVO.java b/engine/schema/src/main/java/com/cloud/storage/VMTemplateVO.java index 3bbebf9ba5f..61df40e50d8 100644 --- a/engine/schema/src/main/java/com/cloud/storage/VMTemplateVO.java +++ b/engine/schema/src/main/java/com/cloud/storage/VMTemplateVO.java @@ -152,6 +152,9 @@ public class VMTemplateVO implements VirtualMachineTemplate { @Column(name = "parent_template_id") private Long parentTemplateId; + @Column(name = "deploy_as_is") + private boolean deployAsIs; + @Override public String getUniqueName() { return uniqueName; @@ -193,7 +196,8 @@ public class VMTemplateVO implements VirtualMachineTemplate { } public VMTemplateVO(long id, String name, ImageFormat format, boolean isPublic, boolean featured, boolean isExtractable, TemplateType type, String url, boolean requiresHvm, int bits, long accountId, String cksum, String displayText, boolean enablePassword, long guestOSId, boolean bootable, - HypervisorType hyperType, String templateTag, Map details, boolean sshKeyEnabled, boolean isDynamicallyScalable, boolean directDownload) { + HypervisorType hyperType, String templateTag, Map details, boolean sshKeyEnabled, boolean isDynamicallyScalable, boolean directDownload, + boolean deployAsIs) { this(id, name, format, @@ -218,6 +222,7 @@ public class VMTemplateVO implements VirtualMachineTemplate { dynamicallyScalable = isDynamicallyScalable; state = State.Active; this.directDownload = directDownload; + this.deployAsIs = deployAsIs; } public static VMTemplateVO createPreHostIso(Long id, String uniqueName, String name, ImageFormat format, boolean isPublic, boolean featured, TemplateType type, @@ -635,4 +640,12 @@ public class VMTemplateVO implements VirtualMachineTemplate { public void setParentTemplateId(Long parentTemplateId) { this.parentTemplateId = parentTemplateId; } + + @Override public boolean isDeployAsIs() { + return deployAsIs; + } + + public void setDeployAsIs(boolean deployAsIs) { + this.deployAsIs = deployAsIs; + } } diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql b/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql index 01969d6e215..788346d7064 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql @@ -261,6 +261,116 @@ CREATE VIEW `cloud`.`storage_pool_view` AS AND (`async_job`.`instance_type` = 'StoragePool') AND (`async_job`.`job_status` = 0)))); +-- Add passthrough instruction for appliance deployments +ALTER TABLE `cloud`.`vm_template` ADD COLUMN `deploy_as_is` tinyint(1) NOT NULL DEFAULT '0' COMMENT 'True if the template should be deployed with disks and networks as defined by OVF'; + +-- Changes to template_view for both deploying multidisk OVA/vApp as is +DROP VIEW IF EXISTS `cloud`.`template_view`; +CREATE VIEW `cloud`.`template_view` AS + SELECT + `vm_template`.`id` AS `id`, + `vm_template`.`uuid` AS `uuid`, + `vm_template`.`unique_name` AS `unique_name`, + `vm_template`.`name` AS `name`, + `vm_template`.`public` AS `public`, + `vm_template`.`featured` AS `featured`, + `vm_template`.`type` AS `type`, + `vm_template`.`hvm` AS `hvm`, + `vm_template`.`bits` AS `bits`, + `vm_template`.`url` AS `url`, + `vm_template`.`format` AS `format`, + `vm_template`.`created` AS `created`, + `vm_template`.`checksum` AS `checksum`, + `vm_template`.`display_text` AS `display_text`, + `vm_template`.`enable_password` AS `enable_password`, + `vm_template`.`dynamically_scalable` AS `dynamically_scalable`, + `vm_template`.`state` AS `template_state`, + `vm_template`.`guest_os_id` AS `guest_os_id`, + `guest_os`.`uuid` AS `guest_os_uuid`, + `guest_os`.`display_name` AS `guest_os_name`, + `vm_template`.`bootable` AS `bootable`, + `vm_template`.`prepopulate` AS `prepopulate`, + `vm_template`.`cross_zones` AS `cross_zones`, + `vm_template`.`hypervisor_type` AS `hypervisor_type`, + `vm_template`.`extractable` AS `extractable`, + `vm_template`.`template_tag` AS `template_tag`, + `vm_template`.`sort_key` AS `sort_key`, + `vm_template`.`removed` AS `removed`, + `vm_template`.`enable_sshkey` AS `enable_sshkey`, + `parent_template`.`id` AS `parent_template_id`, + `parent_template`.`uuid` AS `parent_template_uuid`, + `source_template`.`id` AS `source_template_id`, + `source_template`.`uuid` AS `source_template_uuid`, + `account`.`id` AS `account_id`, + `account`.`uuid` AS `account_uuid`, + `account`.`account_name` AS `account_name`, + `account`.`type` AS `account_type`, + `domain`.`id` AS `domain_id`, + `domain`.`uuid` AS `domain_uuid`, + `domain`.`name` AS `domain_name`, + `domain`.`path` AS `domain_path`, + `projects`.`id` AS `project_id`, + `projects`.`uuid` AS `project_uuid`, + `projects`.`name` AS `project_name`, + `data_center`.`id` AS `data_center_id`, + `data_center`.`uuid` AS `data_center_uuid`, + `data_center`.`name` AS `data_center_name`, + `launch_permission`.`account_id` AS `lp_account_id`, + `template_store_ref`.`store_id` AS `store_id`, + `image_store`.`scope` AS `store_scope`, + `template_store_ref`.`state` AS `state`, + `template_store_ref`.`download_state` AS `download_state`, + `template_store_ref`.`download_pct` AS `download_pct`, + `template_store_ref`.`error_str` AS `error_str`, + `template_store_ref`.`size` AS `size`, + `template_store_ref`.physical_size AS `physical_size`, + `template_store_ref`.`destroyed` AS `destroyed`, + `template_store_ref`.`created` AS `created_on_store`, + `vm_template_details`.`name` AS `detail_name`, + `vm_template_details`.`value` AS `detail_value`, + `resource_tags`.`id` AS `tag_id`, + `resource_tags`.`uuid` AS `tag_uuid`, + `resource_tags`.`key` AS `tag_key`, + `resource_tags`.`value` AS `tag_value`, + `resource_tags`.`domain_id` AS `tag_domain_id`, + `domain`.`uuid` AS `tag_domain_uuid`, + `domain`.`name` AS `tag_domain_name`, + `resource_tags`.`account_id` AS `tag_account_id`, + `account`.`account_name` AS `tag_account_name`, + `resource_tags`.`resource_id` AS `tag_resource_id`, + `resource_tags`.`resource_uuid` AS `tag_resource_uuid`, + `resource_tags`.`resource_type` AS `tag_resource_type`, + `resource_tags`.`customer` AS `tag_customer`, + CONCAT(`vm_template`.`id`, + '_', + IFNULL(`data_center`.`id`, 0)) AS `temp_zone_pair`, + `vm_template`.`direct_download` AS `direct_download`, + `vm_template`.`deploy_as_is` AS `deploy_as_is` + FROM + (((((((((((((`vm_template` + JOIN `guest_os` ON ((`guest_os`.`id` = `vm_template`.`guest_os_id`))) + JOIN `account` ON ((`account`.`id` = `vm_template`.`account_id`))) + JOIN `domain` ON ((`domain`.`id` = `account`.`domain_id`))) + LEFT JOIN `projects` ON ((`projects`.`project_account_id` = `account`.`id`))) + LEFT JOIN `vm_template_details` ON ((`vm_template_details`.`template_id` = `vm_template`.`id`))) + LEFT JOIN `vm_template` `source_template` ON ((`source_template`.`id` = `vm_template`.`source_template_id`))) + LEFT JOIN `template_store_ref` ON (((`template_store_ref`.`template_id` = `vm_template`.`id`) + AND (`template_store_ref`.`store_role` = 'Image') + AND (`template_store_ref`.`destroyed` = 0)))) + LEFT JOIN `vm_template` `parent_template` ON ((`parent_template`.`id` = `vm_template`.`parent_template_id`))) + LEFT JOIN `image_store` ON ((ISNULL(`image_store`.`removed`) + AND (`template_store_ref`.`store_id` IS NOT NULL) + AND (`image_store`.`id` = `template_store_ref`.`store_id`)))) + LEFT JOIN `template_zone_ref` ON (((`template_zone_ref`.`template_id` = `vm_template`.`id`) + AND ISNULL(`template_store_ref`.`store_id`) + AND ISNULL(`template_zone_ref`.`removed`)))) + LEFT JOIN `data_center` ON (((`image_store`.`data_center_id` = `data_center`.`id`) + OR (`template_zone_ref`.`zone_id` = `data_center`.`id`)))) + LEFT JOIN `launch_permission` ON ((`launch_permission`.`template_id` = `vm_template`.`id`))) + LEFT JOIN `resource_tags` ON (((`resource_tags`.`resource_id` = `vm_template`.`id`) + AND ((`resource_tags`.`resource_type` = 'Template') + OR (`resource_tags`.`resource_type` = 'ISO'))))); + -- Add mincpu, maxcpu, minmemory and maxmemory to the view supporting constrained offerings DROP VIEW IF EXISTS `cloud`.`service_offering_view`; CREATE VIEW `cloud`.`service_offering_view` AS diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java index a10aa203ebb..ed9359d952a 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java @@ -412,7 +412,7 @@ public class TemplateServiceImpl implements TemplateService { VirtualMachineTemplate.Event event = VirtualMachineTemplate.Event.OperationSucceeded; // For multi-disk OVA, check and create data disk templates if (tmplt.getFormat().equals(ImageFormat.OVA)) { - if (!createOvaDataDiskTemplates(_templateFactory.getTemplate(tmlpt.getId(), store), true)) { + if (!createOvaDataDiskTemplates(_templateFactory.getTemplate(tmlpt.getId(), store), tmplt.isDeployAsIs())) { event = VirtualMachineTemplate.Event.OperationFailed; } } @@ -710,7 +710,7 @@ public class TemplateServiceImpl implements TemplateService { // For multi-disk OVA, check and create data disk templates if (template.getFormat().equals(ImageFormat.OVA)) { - if (!createOvaDataDiskTemplates(template, true)) { + if (!createOvaDataDiskTemplates(template, template.isDeployAsIs())) { template.processEvent(ObjectInDataStoreStateMachine.Event.OperationFailed); result.setResult(callbackResult.getResult()); if (parentCallback != null) { @@ -799,7 +799,7 @@ public class TemplateServiceImpl implements TemplateService { String templateName = dataDiskTemplate.isIso() ? dataDiskTemplate.getPath().substring(dataDiskTemplate.getPath().lastIndexOf(File.separator) + 1) : template.getName() + suffix + diskCount; VMTemplateVO templateVO = new VMTemplateVO(templateId, templateName, format, false, false, false, ttype, template.getUrl(), template.requiresHvm(), template.getBits(), template.getAccountId(), null, templateName, false, guestOsId, false, template.getHypervisorType(), null, - null, false, false, false); + null, false, false, false, false); if (dataDiskTemplate.isIso()){ templateVO.setUniqueName(templateName); } diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/TemplateObject.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/TemplateObject.java index 6d7ebebdc1d..b7a44cd4f08 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/TemplateObject.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/TemplateObject.java @@ -376,10 +376,10 @@ public class TemplateObject implements TemplateInfo { @Override public boolean isDeployAsIs() { - if (getHypervisorType() == HypervisorType.VMware) { - return true; + if (this.imageVO == null) { + return false; } - return false; + return this.imageVO.isDeployAsIs(); } public void setInstallPath(String installPath) { diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java index 340347fc6f6..37a7985ce6f 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java @@ -32,7 +32,6 @@ import java.util.stream.Collectors; import javax.inject.Inject; -import com.cloud.hypervisor.Hypervisor; import com.cloud.storage.Upload; import org.apache.cloudstack.storage.image.deployasis.DeployAsIsHelper; import org.apache.log4j.Logger; @@ -204,7 +203,7 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver { TemplateDataStoreVO tmpltStoreVO = _templateStoreDao.findByStoreTemplate(store.getId(), obj.getId()); if (tmpltStoreVO != null) { if (tmpltStoreVO.getDownloadState() == VMTemplateStorageResourceAssoc.Status.DOWNLOADED) { - if (template.getHypervisorType() == Hypervisor.HypervisorType.VMware) { + if (template.isDeployAsIs()) { boolean persistDeployAsIs = deployAsIsHelper.persistTemplateDeployAsIsDetails(template.getId(), answer, tmpltStoreVO); if (!persistDeployAsIs) { LOGGER.info("Failed persisting deploy-as-is template details for template " + template.getName()); diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java index d40b30903ee..45509c51c43 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java @@ -21,6 +21,7 @@ import java.util.Date; import javax.inject.Inject; import com.cloud.storage.MigrationOptions; +import com.cloud.storage.VMTemplateVO; import com.cloud.storage.VolumeDetailVO; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VolumeDetailsDao; @@ -445,7 +446,8 @@ public class VolumeObject implements VolumeInfo { @Override public boolean isDeployAsIs() { - return getHypervisorType() == HypervisorType.VMware; + VMTemplateVO template = templateDao.findById(getTemplateId()); + return template != null && template.isDeployAsIs(); } @Override diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java index 6e301985f83..d48a5d9b101 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java @@ -598,7 +598,7 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co private VMTemplateVO createVMTemplateRecord(String vmInternalName, long guestOsId, long accountId) { Long nextTemplateId = vmTemplateDao.getNextInSequence(Long.class, "id"); VMTemplateVO templateVO = new VMTemplateVO(nextTemplateId, "Imported-from-" + vmInternalName, Storage.ImageFormat.OVA, false, false, false, Storage.TemplateType.USER, null, - false, 64, accountId, null, "Template imported from VM " + vmInternalName, false, guestOsId, false, HypervisorType.VMware, null, null, false, false, false); + false, 64, accountId, null, "Template imported from VM " + vmInternalName, false, guestOsId, false, HypervisorType.VMware, null, null, false, false, false, false); return vmTemplateDao.persist(templateVO); } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java index 0e0dd8c56b1..c12c5c20e2d 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java @@ -105,7 +105,7 @@ class VmwareVmImplementer { VirtualMachineTO implement(VirtualMachineProfile vm, VirtualMachineTO to, long clusterId) { to.setBootloader(VirtualMachineTemplate.BootloaderType.HVM); - boolean deployAsIs = true; + boolean deployAsIs = vm.getTemplate().isDeployAsIs(); HostVO host = hostDao.findById(vm.getVirtualMachine().getHostId()); Map details = to.getDetails(); if (details == null) diff --git a/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDaoImpl.java index a8ae39fc6eb..92072c7a3c5 100644 --- a/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDaoImpl.java @@ -253,7 +253,7 @@ public class TemplateJoinDaoImpl extends GenericDaoBaseWithTagInformation deployAsIsDetails = templateDeployAsIsDetailsDao.listDetails(template.getId()); for (TemplateDeployAsIsDetailVO deployAsIsDetailVO : deployAsIsDetails) { if (deployAsIsDetailVO.getName().startsWith(DeployAsIsConstants.HARDWARE_ITEM_PREFIX)) { diff --git a/server/src/main/java/com/cloud/api/query/vo/TemplateJoinVO.java b/server/src/main/java/com/cloud/api/query/vo/TemplateJoinVO.java index 25e3b0b5ff5..91bb76336cc 100644 --- a/server/src/main/java/com/cloud/api/query/vo/TemplateJoinVO.java +++ b/server/src/main/java/com/cloud/api/query/vo/TemplateJoinVO.java @@ -231,6 +231,9 @@ public class TemplateJoinVO extends BaseViewWithTagInformationVO implements Cont @Column(name = "direct_download") private boolean directDownload; + @Column(name = "deploy_as_is") + private boolean deployAsIs; + public TemplateJoinVO() { } @@ -490,6 +493,10 @@ public class TemplateJoinVO extends BaseViewWithTagInformationVO implements Cont return directDownload; } + public boolean isDeployAsIs() { + return deployAsIs; + } + public Object getParentTemplateId() { return parentTemplateId; } diff --git a/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java b/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java index 51d589fcd81..eb99a8eefe3 100644 --- a/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java +++ b/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java @@ -561,7 +561,7 @@ StateListener { * Display storage in the logs by default if the template is not deploy-as-is. */ private boolean getDisplayStorageFromVmProfile(VirtualMachineProfile vmProfile) { - return vmProfile.getHypervisorType() != HypervisorType.VMware || vmProfile.getType() != VirtualMachine.Type.User; + return vmProfile == null || vmProfile.getTemplate() == null || !vmProfile.getTemplate().isDeployAsIs(); } @Override diff --git a/server/src/main/java/com/cloud/storage/ImageStoreUploadMonitorImpl.java b/server/src/main/java/com/cloud/storage/ImageStoreUploadMonitorImpl.java index 5174b2613ab..1ce5b362eb9 100755 --- a/server/src/main/java/com/cloud/storage/ImageStoreUploadMonitorImpl.java +++ b/server/src/main/java/com/cloud/storage/ImageStoreUploadMonitorImpl.java @@ -412,7 +412,7 @@ public class ImageStoreUploadMonitorImpl extends ManagerBase implements ImageSto if (tmpTemplate.getFormat().equals(Storage.ImageFormat.OVA)) { final DataStore store = dataStoreManager.getDataStore(templateDataStore.getDataStoreId(), templateDataStore.getDataStoreRole()); final TemplateInfo templateInfo = templateFactory.getTemplate(tmpTemplate.getId(), store); - if (!templateService.createOvaDataDiskTemplates(templateInfo, true)) { + if (!templateService.createOvaDataDiskTemplates(templateInfo, template.isDeployAsIs())) { tmpTemplateDataStore.setDownloadState(VMTemplateStorageResourceAssoc.Status.ABANDONED); tmpTemplateDataStore.setState(State.Failed); stateMachine.transitTo(tmpTemplate, VirtualMachineTemplate.Event.OperationFailed, null, _templateDao); diff --git a/server/src/main/java/com/cloud/storage/TemplateProfile.java b/server/src/main/java/com/cloud/storage/TemplateProfile.java index 304b652a589..b90409480bc 100644 --- a/server/src/main/java/com/cloud/storage/TemplateProfile.java +++ b/server/src/main/java/com/cloud/storage/TemplateProfile.java @@ -52,6 +52,7 @@ public class TemplateProfile { Boolean isDynamicallyScalable; TemplateType templateType; Boolean directDownload; + Boolean deployAsIs; Long size; public TemplateProfile(Long templateId, Long userId, String name, String displayText, Integer bits, Boolean passwordEnabled, Boolean requiresHvm, String url, @@ -95,7 +96,7 @@ public class TemplateProfile { Boolean isPublic, Boolean featured, Boolean isExtractable, ImageFormat format, Long guestOsId, List zoneId, HypervisorType hypervisorType, String accountName, Long domainId, Long accountId, String chksum, Boolean bootable, String templateTag, Map details, - Boolean sshKeyEnabled, Long imageStoreId, Boolean isDynamicallyScalable, TemplateType templateType, Boolean directDownload) { + Boolean sshKeyEnabled, Long imageStoreId, Boolean isDynamicallyScalable, TemplateType templateType, Boolean directDownload, Boolean deployAsIs) { this(templateId, userId, name, @@ -122,6 +123,7 @@ public class TemplateProfile { this.isDynamicallyScalable = isDynamicallyScalable; this.templateType = templateType; this.directDownload = directDownload; + this.deployAsIs = deployAsIs; } public Long getTemplateId() { @@ -331,4 +333,8 @@ public class TemplateProfile { public void setSize(Long size) { this.size = size; } + + public boolean isDeployAsIs() { + return this.deployAsIs; + } } diff --git a/server/src/main/java/com/cloud/template/TemplateAdapterBase.java b/server/src/main/java/com/cloud/template/TemplateAdapterBase.java index 9cd2b6f791c..45a10a8c4b3 100644 --- a/server/src/main/java/com/cloud/template/TemplateAdapterBase.java +++ b/server/src/main/java/com/cloud/template/TemplateAdapterBase.java @@ -265,7 +265,7 @@ public abstract class TemplateAdapterBase extends AdapterBase implements Templat CallContext.current().setEventDetails("Id: " + id + " name: " + name); return new TemplateProfile(id, userId, name, displayText, bits, passwordEnabled, requiresHVM, url, isPublic, featured, isExtractable, imgfmt, guestOSId, zoneIdList, hypervisorType, templateOwner.getAccountName(), templateOwner.getDomainId(), templateOwner.getAccountId(), chksum, bootable, templateTag, details, - sshkeyEnabled, null, isDynamicallyScalable, templateType, directDownload); + sshkeyEnabled, null, isDynamicallyScalable, templateType, directDownload, deployAsIs); } @@ -376,7 +376,7 @@ public abstract class TemplateAdapterBase extends AdapterBase implements Templat new VMTemplateVO(profile.getTemplateId(), profile.getName(), profile.getFormat(), profile.isPublic(), profile.isFeatured(), profile.isExtractable(), profile.getTemplateType(), profile.getUrl(), profile.isRequiresHVM(), profile.getBits(), profile.getAccountId(), profile.getCheckSum(), profile.getDisplayText(), profile.isPasswordEnabled(), profile.getGuestOsId(), profile.isBootable(), profile.getHypervisorType(), - profile.getTemplateTag(), profile.getDetails(), profile.isSshKeyEnabled(), profile.IsDynamicallyScalable(), profile.isDirectDownload()); + profile.getTemplateTag(), profile.getDetails(), profile.isSshKeyEnabled(), profile.IsDynamicallyScalable(), profile.isDirectDownload(), profile.isDeployAsIs()); template.setState(initialState); if (profile.isDirectDownload()) { diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index 519072272dd..c58b2f1ad9a 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -1898,7 +1898,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, } privateTemplate = new VMTemplateVO(nextTemplateId, name, ImageFormat.RAW, isPublic, featured, isExtractable, TemplateType.USER, null, requiresHvmValue, bitsValue, templateOwner.getId(), null, description, - passwordEnabledValue, guestOS.getId(), true, hyperType, templateTag, cmd.getDetails(), sshKeyEnabledValue, isDynamicScalingEnabled, false); + passwordEnabledValue, guestOS.getId(), true, hyperType, templateTag, cmd.getDetails(), sshKeyEnabledValue, isDynamicScalingEnabled, false, false); if (sourceTemplateId != null) { if (s_logger.isDebugEnabled()) { diff --git a/server/src/test/java/com/cloud/vm/DeploymentPlanningManagerImplTest.java b/server/src/test/java/com/cloud/vm/DeploymentPlanningManagerImplTest.java index 71a6935a7d8..d356570b633 100644 --- a/server/src/test/java/com/cloud/vm/DeploymentPlanningManagerImplTest.java +++ b/server/src/test/java/com/cloud/vm/DeploymentPlanningManagerImplTest.java @@ -29,6 +29,7 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; import com.cloud.host.Host; +import com.cloud.storage.VMTemplateVO; import com.cloud.storage.dao.VMTemplateDao; import org.apache.cloudstack.affinity.dao.AffinityGroupDomainMapDao; import org.junit.Before; @@ -166,6 +167,10 @@ public class DeploymentPlanningManagerImplTest { Mockito.when(_plannerHostReserveDao.findById(Matchers.anyLong())).thenReturn(reservationVO); Mockito.when(_affinityGroupVMMapDao.countAffinityGroupsForVm(Matchers.anyLong())).thenReturn(0L); + VMTemplateVO template = Mockito.mock(VMTemplateVO.class); + Mockito.when(template.isDeployAsIs()).thenReturn(false); + Mockito.when(templateDao.findById(Mockito.anyLong())).thenReturn(template); + VMInstanceVO vm = new VMInstanceVO(); Mockito.when(vmProfile.getVirtualMachine()).thenReturn(vm); From 897cc4bdba3d9d8d79a877c0389ff7a41746b8b0 Mon Sep 17 00:00:00 2001 From: nvazquez Date: Fri, 2 Oct 2020 04:46:57 -0300 Subject: [PATCH 118/164] Fix nested virt marvin test --- .../vmware/resource/VmwareResource.java | 46 +++++----- .../smoke/test_nested_virtualization.py | 86 +++++++++++-------- .../vmware/mo/VirtualMachineMO.java | 8 +- 3 files changed, 74 insertions(+), 66 deletions(-) diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index a3990bb4f90..4a851a49f47 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -1942,30 +1942,28 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa VmwareHelper.setBasicVmConfig(vmConfigSpec, vmSpec.getCpus(), vmSpec.getMaxSpeed(), getReservedCpuMHZ(vmSpec), (int) (vmSpec.getMaxRam() / (1024 * 1024)), getReservedMemoryMb(vmSpec), guestOsId, vmSpec.getLimitCpuUse(), deployAsIs); - if (!deployAsIs) { - // Check for multi-cores per socket settings - int numCoresPerSocket = 1; - String coresPerSocket = vmSpec.getDetails().get(VmDetailConstants.CPU_CORE_PER_SOCKET); - if (coresPerSocket != null) { - String apiVersion = HypervisorHostHelper.getVcenterApiVersion(vmMo.getContext()); - // Property 'numCoresPerSocket' is supported since vSphere API 5.0 - if (apiVersion.compareTo("5.0") >= 0) { - numCoresPerSocket = NumbersUtil.parseInt(coresPerSocket, 1); - vmConfigSpec.setNumCoresPerSocket(numCoresPerSocket); - } + // Check for multi-cores per socket settings + int numCoresPerSocket = 1; + String coresPerSocket = vmSpec.getDetails().get(VmDetailConstants.CPU_CORE_PER_SOCKET); + if (coresPerSocket != null) { + String apiVersion = HypervisorHostHelper.getVcenterApiVersion(vmMo.getContext()); + // Property 'numCoresPerSocket' is supported since vSphere API 5.0 + if (apiVersion.compareTo("5.0") >= 0) { + numCoresPerSocket = NumbersUtil.parseInt(coresPerSocket, 1); + vmConfigSpec.setNumCoresPerSocket(numCoresPerSocket); } + } - // Check for hotadd settings - vmConfigSpec.setMemoryHotAddEnabled(vmMo.isMemoryHotAddSupported(guestOsId)); + // Check for hotadd settings + vmConfigSpec.setMemoryHotAddEnabled(vmMo.isMemoryHotAddSupported(guestOsId)); - String hostApiVersion = ((HostMO) hyperHost).getHostAboutInfo().getApiVersion(); - if (numCoresPerSocket > 1 && hostApiVersion.compareTo("5.0") < 0) { - s_logger.warn("Dynamic scaling of CPU is not supported for Virtual Machines with multi-core vCPUs in case of ESXi hosts 4.1 and prior. Hence CpuHotAdd will not be" - + " enabled for Virtual Machine: " + vmInternalCSName); - vmConfigSpec.setCpuHotAddEnabled(false); - } else { - vmConfigSpec.setCpuHotAddEnabled(vmMo.isCpuHotAddSupported(guestOsId)); - } + String hostApiVersion = ((HostMO) hyperHost).getHostAboutInfo().getApiVersion(); + if (numCoresPerSocket > 1 && hostApiVersion.compareTo("5.0") < 0) { + s_logger.warn("Dynamic scaling of CPU is not supported for Virtual Machines with multi-core vCPUs in case of ESXi hosts 4.1 and prior. Hence CpuHotAdd will not be" + + " enabled for Virtual Machine: " + vmInternalCSName); + vmConfigSpec.setCpuHotAddEnabled(false); + } else { + vmConfigSpec.setCpuHotAddEnabled(vmMo.isCpuHotAddSupported(guestOsId)); } configNestedHVSupport(vmMo, vmSpec, vmConfigSpec); @@ -2479,11 +2477,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } private String getGuestOsIdFromVmSpec(VirtualMachineTO vmSpec, boolean deployAsIs) { - String guestOsId = null; - if (!deployAsIs) { - guestOsId = translateGuestOsIdentifier(vmSpec.getArch(), vmSpec.getOs(), vmSpec.getPlatformEmulator()).value(); - } - return guestOsId; + return translateGuestOsIdentifier(vmSpec.getArch(), vmSpec.getOs(), vmSpec.getPlatformEmulator()).value(); } private Pair getControllerInfoFromVmSpec(VirtualMachineTO vmSpec) throws CloudRuntimeException { diff --git a/test/integration/smoke/test_nested_virtualization.py b/test/integration/smoke/test_nested_virtualization.py index c10dd2f98a4..43313244f41 100644 --- a/test/integration/smoke/test_nested_virtualization.py +++ b/test/integration/smoke/test_nested_virtualization.py @@ -97,51 +97,61 @@ class TestNestedVirtualization(cloudstackTestCase): config_update = Configurations.update(self.apiclient, "vmware.nested.virtualization.perVM", "true") self.logger.debug("Updated global setting vmware.nested.virtualization.perVM to true") rollback_nv_per_vm = True - - # 2) Deploy a vm - virtual_machine = VirtualMachine.create( - self.apiclient, - self.services["small"], - accountid=self.account.name, - domainid=self.account.domainid, - serviceofferingid=self.service_offering.id, - mode=self.services['mode'] - ) - self.assert_(virtual_machine is not None, "VM failed to deploy") - self.assert_(virtual_machine.state == 'Running', "VM is not running") - self.logger.debug("Deployed vm: %s" % virtual_machine.id) - - isolated_network = Network.create( - self.apiclient, - self.services["isolated_network"], - self.account.name, - self.account.domainid, - networkofferingid=self.isolated_network_offering.id) - virtual_machine.add_nic(self.apiclient, isolated_network.id) - - # 3) SSH into vm - ssh_client = virtual_machine.get_ssh_client() + try: + # 2) Deploy a vm + virtual_machine = VirtualMachine.create( + self.apiclient, + self.services["small"], + accountid=self.account.name, + domainid=self.account.domainid, + serviceofferingid=self.service_offering.id, + mode=self.services['mode'] + ) + self.assert_(virtual_machine is not None, "VM failed to deploy") + self.assert_(virtual_machine.state == 'Running', "VM is not running") + self.logger.debug("Deployed vm: %s" % virtual_machine.id) - if ssh_client: - # run ping test - result = ssh_client.execute("cat /proc/cpuinfo | grep flags") - self.logger.debug(result) - else: - self.fail("Failed to setup ssh connection to %s" % virtual_machine.public_ip) - - # 4) Revert configurations, if needed + isolated_network = Network.create( + self.apiclient, + self.services["isolated_network"], + self.account.name, + self.account.domainid, + networkofferingid=self.isolated_network_offering.id) + + virtual_machine.stop(self.apiclient) + virtual_machine.add_nic(self.apiclient, isolated_network.id) + virtual_machine.start(self.apiclient) + + # 3) SSH into vm + ssh_client = virtual_machine.get_ssh_client() + + if ssh_client: + # run ping test + result = ssh_client.execute("cat /proc/cpuinfo | grep flags") + self.logger.debug(result) + else: + self.fail("Failed to setup ssh connection to %s" % virtual_machine.public_ip) + + # 4) Revert configurations, if needed + self.rollback_nested_configurations(rollback_nv, rollback_nv_per_vm) + + #5) Check for CPU flags: vmx for Intel and svm for AMD indicates nested virtualization is enabled + self.assert_(result is not None, "Empty result for CPU flags") + res = str(result) + self.assertTrue('vmx' in res or 'svm' in res or 'lm' in res) + except Exception as e: + self.debug('Error=%s' % e) + self.rollback_nested_configurations(rollback_nv, rollback_nv_per_vm) + raise e + + def rollback_nested_configurations(self, rollback_nv, rollback_nv_per_vm): if rollback_nv: config_update = Configurations.update(self.apiclient, "vmware.nested.virtualization", "false") self.logger.debug("Reverted global setting vmware.nested.virtualization back to false") if rollback_nv_per_vm: - config_update = Configurations.update(self.apiclient, "vmware.nested.virtualization", "false") + config_update = Configurations.update(self.apiclient, "vmware.nested.virtualization.perVM", "false") self.logger.debug("Reverted global setting vmware.nested.virtualization.perVM back to false") - - #5) Check for CPU flags: vmx for Intel and svm for AMD indicates nested virtualization is enabled - self.assert_(result is not None, "Empty result for CPU flags") - res = str(result) - self.assertTrue('vmx' in res or 'svm' in res) @classmethod def tearDownClass(cls): diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java index 44eb47b93d1..2135e2b1816 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java @@ -227,11 +227,15 @@ public class VirtualMachineMO extends BaseMO { s_logger.info("msg id: " + msg.getId()); s_logger.info("msg text: " + msg.getText()); } + String logMsg = "Found that VM has a pending question that we need to answer programmatically, question id: " + msg.getId(); if ("msg.uuid.altered".equalsIgnoreCase(msg.getId())) { - s_logger.info("Found that VM has a pending question that we need to answer programmatically, question id: " + msg.getId() - + ", we will automatically answer as 'moved it' to address out of band HA for the VM"); + s_logger.info(logMsg + ", we will automatically answer as 'moved it' to address out of band HA for the VM"); vmMo.answerVM(question.getId(), "1"); break; + } else if (msg.getId().equalsIgnoreCase("msg.cpuid.noVHVQuestion")) { + s_logger.info(logMsg + ", automatically answering 'yes'"); + vmMo.answerVM(question.getId(), "0"); + break; } } } From 588b7a1c90b7d7921dfcdc0a346f45ebc854f1d8 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Mon, 5 Oct 2020 15:01:00 +0530 Subject: [PATCH 119/164] Cleanup volume wrapper VM(ROOT-xxx) while deploying VM in case of any failure. since the same name will be used on retries and that will eventually fails saying volume wrapper VM already exists. --- .../resource/VmwareStorageProcessor.java | 82 ++++++++++--------- 1 file changed, 45 insertions(+), 37 deletions(-) diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java index fe58da56b1b..87cde221ea0 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -910,55 +910,63 @@ public class VmwareStorageProcessor implements StorageProcessor { private String createVMFolderWithVMName(VmwareContext context, VmwareHypervisorHost hyperHost, TemplateObjectTO template, VirtualMachineMO vmTemplate, VolumeObjectTO volume, DatacenterMO dcMo, DatastoreMO dsMo, String searchExcludedFolders) throws Exception { - ManagedObjectReference morDatastore = dsMo.getMor(); - ManagedObjectReference morPool = hyperHost.getHyperHostOwnerResourcePool(); - ManagedObjectReference morCluster = hyperHost.getHyperHostCluster(); String vmdkName = volume.getName(); - if (template.getSize() != null){ - _fullCloneFlag = volume.getSize() > template.getSize() ? true : _fullCloneFlag; - } - if (!_fullCloneFlag) { - createVMLinkedClone(vmTemplate, dcMo, vmdkName, morDatastore, morPool); - } else { - createVMFullClone(vmTemplate, dcMo, dsMo,vmdkName, morDatastore, morPool); - } + try { + ManagedObjectReference morDatastore = dsMo.getMor(); + ManagedObjectReference morPool = hyperHost.getHyperHostOwnerResourcePool(); + ManagedObjectReference morCluster = hyperHost.getHyperHostCluster(); + if (template.getSize() != null){ + _fullCloneFlag = volume.getSize() > template.getSize() ? true : _fullCloneFlag; + } + if (!_fullCloneFlag) { + createVMLinkedClone(vmTemplate, dcMo, vmdkName, morDatastore, morPool); + } else { + createVMFullClone(vmTemplate, dcMo, dsMo, vmdkName, morDatastore, morPool); + } - VirtualMachineMO vmMo = new ClusterMO(context, morCluster).findVmOnHyperHost(vmdkName); - assert (vmMo != null); + VirtualMachineMO vmMo = new ClusterMO(context, morCluster).findVmOnHyperHost(vmdkName); + assert (vmMo != null); - String vmdkFileBaseName = vmMo.getVmdkFileBaseNames().get(0); - s_logger.info("Move volume out of volume-wrapper VM " + vmdkFileBaseName); - String[] vmwareLayoutFilePair = VmwareStorageLayoutHelper.getVmdkFilePairDatastorePath(dsMo, vmdkName, vmdkFileBaseName, VmwareStorageLayoutType.VMWARE, !_fullCloneFlag); - String[] legacyCloudStackLayoutFilePair = VmwareStorageLayoutHelper.getVmdkFilePairDatastorePath(dsMo, vmdkName, vmdkFileBaseName, VmwareStorageLayoutType.CLOUDSTACK_LEGACY, !_fullCloneFlag); + String vmdkFileBaseName = vmMo.getVmdkFileBaseNames().get(0); + s_logger.info("Move volume out of volume-wrapper VM " + vmdkFileBaseName); + String[] vmwareLayoutFilePair = VmwareStorageLayoutHelper.getVmdkFilePairDatastorePath(dsMo, vmdkName, vmdkFileBaseName, VmwareStorageLayoutType.VMWARE, !_fullCloneFlag); + String[] legacyCloudStackLayoutFilePair = VmwareStorageLayoutHelper.getVmdkFilePairDatastorePath(dsMo, vmdkName, vmdkFileBaseName, VmwareStorageLayoutType.CLOUDSTACK_LEGACY, !_fullCloneFlag); - for (int i=0; i Date: Tue, 6 Oct 2020 10:47:01 +0530 Subject: [PATCH 120/164] Handle detach volume of datastore cluster if the volume name has changed at vCenter level and reconsile the chaininfo --- .../orchestration/VolumeOrchestrator.java | 6 +- .../datastore/db/PrimaryDataStoreDao.java | 1 - .../datastore/db/PrimaryDataStoreDaoImpl.java | 11 -- .../vmware/resource/VmwareResource.java | 20 ++- .../resource/VmwareStorageProcessor.java | 122 ++++++++++++++---- .../cloud/storage/VolumeApiServiceImpl.java | 19 ++- .../java/com/cloud/vm/UserVmManagerImpl.java | 16 ++- 7 files changed, 144 insertions(+), 51 deletions(-) diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java index b4d662d449c..6254a999031 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java @@ -1734,9 +1734,9 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati vol.setPath(path); vol.setChainInfo(chainInfo); if (updatedDataStoreUUID != null) { - List pools = _storagePoolDao.listPoolsByLikePath(updatedDataStoreUUID); - if (pools != null && !pools.isEmpty()) { - vol.setPoolId(pools.get(0).getId()); + StoragePoolVO pool = _storagePoolDao.findByUuid(updatedDataStoreUUID); + if (pool != null) { + vol.setPoolId(pool.getId()); } } _volsDao.update(volumeId, vol); diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java index 0cbd0e14f28..59fe3f14d39 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java @@ -128,5 +128,4 @@ public interface PrimaryDataStoreDao extends GenericDao { Integer countAll(); - public List listPoolsByLikePath(String path); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java index 315f698c099..7830df57204 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java @@ -56,7 +56,6 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase private final SearchBuilder DeleteLvmSearch; private final SearchBuilder DcLocalStorageSearch; private final GenericSearchBuilder StatusCountSearch; - private final SearchBuilder PathLikeSearch; @Inject private StoragePoolDetailsDao _detailsDao; @@ -134,16 +133,6 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase DcLocalStorageSearch.and("scope", DcLocalStorageSearch.entity().getScope(), SearchCriteria.Op.EQ); DcLocalStorageSearch.done(); - PathLikeSearch = createSearchBuilder(); - PathLikeSearch.and("path", PathLikeSearch.entity().getPath(), SearchCriteria.Op.LIKE); - PathLikeSearch.done(); - } - - @Override - public List listPoolsByLikePath(String path) { - SearchCriteria sc = PathLikeSearch.create(); - sc.setParameters("path", "%" + path + "%"); - return listBy(sc); } @Override diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 4a851a49f47..d2b082ae393 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -2127,12 +2127,23 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } if (vol.getDetails().get(DiskTO.PROTOCOL_TYPE) != null && vol.getDetails().get(DiskTO.PROTOCOL_TYPE).equalsIgnoreCase("DatastoreCluster")) { - if (diskInfoBuilder != null && matchingExistingDisk == null) { + if (diskInfoBuilder != null && matchingExistingDisk != null) { + String[] diskChain = matchingExistingDisk.getDiskChain(); + if (diskChain != null && diskChain.length > 0) { + DatastoreFile file = new DatastoreFile(diskChain[0]); + if (!file.getFileBaseName().equalsIgnoreCase(volumeTO.getPath())) { + if (s_logger.isInfoEnabled()) + s_logger.info("Detected disk-chain top file change on volume: " + volumeTO.getId() + " " + volumeTO.getPath() + " -> " + file.getFileBaseName()); + volumeTO.setPath(file.getFileBaseName()); + } + } DatastoreMO diskDatastoreMofromVM = getDataStoreWhereDiskExists(hyperHost, context, diskInfoBuilder, vol, diskDatastores); if (diskDatastoreMofromVM != null) { String actualPoolUuid = diskDatastoreMofromVM.getCustomFieldValue(CustomFieldConstants.CLOUD_UUID); - if (!actualPoolUuid.equalsIgnoreCase(primaryStore.getUuid())) { + if (actualPoolUuid != null && !actualPoolUuid.equalsIgnoreCase(primaryStore.getUuid())) { volumeDsDetails = new Pair<>(diskDatastoreMofromVM.getMor(), diskDatastoreMofromVM); + if (s_logger.isInfoEnabled()) + s_logger.info("Detected datastore uuid change on volume: " + volumeTO.getId() + " " + primaryStore.getUuid() + " -> " + actualPoolUuid); ((PrimaryDataStoreTO)primaryStore).setUuid(actualPoolUuid); } } @@ -3322,8 +3333,9 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa volInSpec.setPath(datastoreVolumePath); } else { volInSpec.setPath(file.getFileBaseName()); - if (!file.getDatastoreName().equals(volumeTO.getDataStore().getUuid())) - volInSpec.setUpdatedDataStoreUUID(file.getDatastoreName()); + if (vol.getDetails().get(DiskTO.PROTOCOL_TYPE) != null && vol.getDetails().get(DiskTO.PROTOCOL_TYPE).equalsIgnoreCase("DatastoreCluster")) { + volInSpec.setUpdatedDataStoreUUID(volumeTO.getDataStore().getUuid()); + } } volInSpec.setChainInfo(_gson.toJson(diskInfo)); } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java index 87cde221ea0..7d5b0ba839d 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -2041,8 +2041,9 @@ public class VmwareStorageProcessor implements StorageProcessor { String storageHost, int storagePort, Map controllerInfo) { VolumeObjectTO volumeTO = (VolumeObjectTO)disk.getData(); DataStoreTO primaryStore = volumeTO.getDataStore(); + String volumePath = volumeTO.getPath(); - String vmdkPath = isManaged ? resource.getVmdkPath(volumeTO.getPath()) : null; + String vmdkPath = isManaged ? resource.getVmdkPath(volumePath) : null; try { VmwareContext context = hostService.getServiceContext(null); @@ -2091,25 +2092,27 @@ public class VmwareStorageProcessor implements StorageProcessor { DatastoreMO dsMo = new DatastoreMO(context, morDs); String datastoreVolumePath; boolean datastoreChangeObserved = false; + boolean volumePathChangeObserved = false; + String chainInfo = null; if (isAttach) { if (isManaged) { datastoreVolumePath = dsMo.getDatastorePath((vmdkPath != null ? vmdkPath : dsMo.getName()) + ".vmdk"); } else { if (dsMo.getDatastoreType().equalsIgnoreCase("VVOL")) { - datastoreVolumePath = VmwareStorageLayoutHelper.getLegacyDatastorePathFromVmdkFileName(dsMo, volumeTO.getPath() + ".vmdk"); + datastoreVolumePath = VmwareStorageLayoutHelper.getLegacyDatastorePathFromVmdkFileName(dsMo, volumePath + ".vmdk"); if (!dsMo.fileExists(datastoreVolumePath)) { - datastoreVolumePath = VmwareStorageLayoutHelper.getVmwareDatastorePathFromVmdkFileName(dsMo, vmName, volumeTO.getPath() + ".vmdk"); + datastoreVolumePath = VmwareStorageLayoutHelper.getVmwareDatastorePathFromVmdkFileName(dsMo, vmName, volumePath + ".vmdk"); } if (!dsMo.folderExists(String.format("[%s]", dsMo.getName()), vmName) || !dsMo.fileExists(datastoreVolumePath)) { - datastoreVolumePath = VmwareStorageLayoutHelper.getVmwareDatastorePathFromVmdkFileName(dsMo, volumeTO.getPath(), volumeTO.getPath() + ".vmdk"); + datastoreVolumePath = VmwareStorageLayoutHelper.getVmwareDatastorePathFromVmdkFileName(dsMo, volumePath, volumePath + ".vmdk"); } - if (!dsMo.folderExists(String.format("[%s]", dsMo.getName()), volumeTO.getPath()) || !dsMo.fileExists(datastoreVolumePath)) { - datastoreVolumePath = dsMo.searchFileInSubFolders(volumeTO.getPath() + ".vmdk", true, null); + if (!dsMo.folderExists(String.format("[%s]", dsMo.getName()), volumePath) || !dsMo.fileExists(datastoreVolumePath)) { + datastoreVolumePath = dsMo.searchFileInSubFolders(volumePath + ".vmdk", true, null); } } else { - datastoreVolumePath = VmwareStorageLayoutHelper.syncVolumeToVmDefaultFolder(dsMo.getOwnerDatacenter().first(), vmName, dsMo, volumeTO.getPath(), VmwareManager.s_vmwareSearchExcludeFolder.value()); + datastoreVolumePath = VmwareStorageLayoutHelper.syncVolumeToVmDefaultFolder(dsMo.getOwnerDatacenter().first(), vmName, dsMo, volumePath, VmwareManager.s_vmwareSearchExcludeFolder.value()); } } } else { @@ -2118,13 +2121,30 @@ public class VmwareStorageProcessor implements StorageProcessor { } else { String datastoreUUID = primaryStore.getUuid(); if (disk.getDetails().get(DiskTO.PROTOCOL_TYPE) != null && disk.getDetails().get(DiskTO.PROTOCOL_TYPE).equalsIgnoreCase("DatastoreCluster")) { - DatastoreMO diskDatastoreMoFromVM = getDiskDatastoreMofromVM(hyperHost, context, vmMo, disk); - if (diskDatastoreMoFromVM != null) { - String actualPoolUuid = diskDatastoreMoFromVM.getCustomFieldValue(CustomFieldConstants.CLOUD_UUID); - if (!actualPoolUuid.equalsIgnoreCase(primaryStore.getUuid())) { - s_logger.warn(String.format("Volume %s found to be in a different storage pool %s", volumeTO.getPath(), actualPoolUuid)); - datastoreChangeObserved = true; - datastoreUUID = actualPoolUuid; + VirtualMachineDiskInfo matchingExistingDisk = getMatchingExistingDisk(hyperHost, context, vmMo, disk); + VirtualMachineDiskInfoBuilder diskInfoBuilder = vmMo.getDiskInfoBuilder(); + if (diskInfoBuilder != null && matchingExistingDisk != null) { + String[] diskChain = matchingExistingDisk.getDiskChain(); + assert (diskChain.length > 0); + DatastoreFile file = new DatastoreFile(diskChain[0]); + if (!file.getFileBaseName().equalsIgnoreCase(volumePath)) { + if (s_logger.isInfoEnabled()) + s_logger.info("Detected disk-chain top file change on volume: " + volumeTO.getId() + " " + volumePath + " -> " + file.getFileBaseName()); + volumePathChangeObserved = true; + volumePath = file.getFileBaseName(); + volumeTO.setPath(volumePath); + chainInfo = _gson.toJson(matchingExistingDisk); + } + + DatastoreMO diskDatastoreMofromVM = getDiskDatastoreMofromVM(hyperHost, context, vmMo, disk, diskInfoBuilder); + if (diskDatastoreMofromVM != null) { + String actualPoolUuid = diskDatastoreMofromVM.getCustomFieldValue(CustomFieldConstants.CLOUD_UUID); + if (!actualPoolUuid.equalsIgnoreCase(primaryStore.getUuid())) { + s_logger.warn(String.format("Volume %s found to be in a different storage pool %s", volumePath, actualPoolUuid)); + datastoreChangeObserved = true; + datastoreUUID = actualPoolUuid; + chainInfo = _gson.toJson(matchingExistingDisk); + } } } } @@ -2135,15 +2155,15 @@ public class VmwareStorageProcessor implements StorageProcessor { } dsMo = new DatastoreMO(context, morDs); - datastoreVolumePath = VmwareStorageLayoutHelper.getLegacyDatastorePathFromVmdkFileName(dsMo, volumeTO.getPath() + ".vmdk"); + datastoreVolumePath = VmwareStorageLayoutHelper.getLegacyDatastorePathFromVmdkFileName(dsMo, volumePath + ".vmdk"); if (!dsMo.fileExists(datastoreVolumePath)) { - datastoreVolumePath = VmwareStorageLayoutHelper.getVmwareDatastorePathFromVmdkFileName(dsMo, vmName, volumeTO.getPath() + ".vmdk"); + datastoreVolumePath = VmwareStorageLayoutHelper.getVmwareDatastorePathFromVmdkFileName(dsMo, vmName, volumePath + ".vmdk"); } if (!dsMo.folderExists(String.format("[%s]", dsMo.getName()), vmName) || !dsMo.fileExists(datastoreVolumePath)) { - datastoreVolumePath = VmwareStorageLayoutHelper.getVmwareDatastorePathFromVmdkFileName(dsMo, volumeTO.getPath(), volumeTO.getPath() + ".vmdk"); + datastoreVolumePath = VmwareStorageLayoutHelper.getVmwareDatastorePathFromVmdkFileName(dsMo, volumePath, volumePath + ".vmdk"); } - if (!dsMo.folderExists(String.format("[%s]", dsMo.getName()), volumeTO.getPath()) || !dsMo.fileExists(datastoreVolumePath)) { - datastoreVolumePath = dsMo.searchFileInSubFolders(volumeTO.getPath() + ".vmdk", true, null); + if (!dsMo.folderExists(String.format("[%s]", dsMo.getName()), volumePath) || !dsMo.fileExists(datastoreVolumePath)) { + datastoreVolumePath = dsMo.searchFileInSubFolders(volumePath + ".vmdk", true, null); } } } @@ -2176,11 +2196,18 @@ public class VmwareStorageProcessor implements StorageProcessor { handleDatastoreAndVmdkDetachManaged(cmd, diskUuid, iScsiName, storageHost, storagePort); } else { if (!dsMo.getDatastoreType().equalsIgnoreCase("VVOL")) { - VmwareStorageLayoutHelper.syncVolumeToRootFolder(dsMo.getOwnerDatacenter().first(), dsMo, volumeTO.getPath(), vmName, VmwareManager.s_vmwareSearchExcludeFolder.value()); + VmwareStorageLayoutHelper.syncVolumeToRootFolder(dsMo.getOwnerDatacenter().first(), dsMo, volumePath, vmName, VmwareManager.s_vmwareSearchExcludeFolder.value()); } } - if (datastoreChangeObserved) - answer.setContextParam("datastoreName", dsMo.getName()); + if (datastoreChangeObserved) { + answer.setContextParam("datastoreName", dsMo.getCustomFieldValue(CustomFieldConstants.CLOUD_UUID)); + answer.setContextParam("chainInfo", chainInfo); + } + + if (volumePathChangeObserved) { + answer.setContextParam("volumePath", volumePath); + answer.setContextParam("chainInfo", chainInfo); + } } return answer; @@ -2206,11 +2233,58 @@ public class VmwareStorageProcessor implements StorageProcessor { } } + private VirtualMachineDiskInfo getMatchingExistingDisk(VmwareHypervisorHost hyperHost, VmwareContext context, VirtualMachineMO vmMo, DiskTO vol) + throws Exception { + VirtualMachineDiskInfoBuilder diskInfoBuilder = vmMo.getDiskInfoBuilder(); + if (diskInfoBuilder != null) { + VolumeObjectTO volume = (VolumeObjectTO) vol.getData(); + + ManagedObjectReference morDs = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, volume.getDataStore().getUuid()); + DatastoreMO dsMo = new DatastoreMO(context, morDs); + + String dsName = dsMo.getName(); + + String diskBackingFileBaseName = volume.getPath(); + + VirtualMachineDiskInfo diskInfo = diskInfoBuilder.getDiskInfoByBackingFileBaseName(diskBackingFileBaseName, dsName); + if (diskInfo != null) { + s_logger.info("Found existing disk info from volume path: " + volume.getPath()); + return diskInfo; + } else { + String chainInfo = volume.getChainInfo(); + if (chainInfo != null) { + VirtualMachineDiskInfo infoInChain = _gson.fromJson(chainInfo, VirtualMachineDiskInfo.class); + if (infoInChain != null) { + String[] disks = infoInChain.getDiskChain(); + if (disks.length > 0) { + for (String diskPath : disks) { + DatastoreFile file = new DatastoreFile(diskPath); + diskInfo = diskInfoBuilder.getDiskInfoByBackingFileBaseName(file.getFileBaseName(), dsName); + if (diskInfo != null) { + s_logger.info("Found existing disk from chain info: " + diskPath); + return diskInfo; + } + } + } + + if (diskInfo == null) { + diskInfo = diskInfoBuilder.getDiskInfoByDeviceBusName(infoInChain.getDiskDeviceBusName()); + if (diskInfo != null) { + s_logger.info("Found existing disk from from chain device bus information: " + infoInChain.getDiskDeviceBusName()); + return diskInfo; + } + } + } + } + } + } + return null; + } + private DatastoreMO getDiskDatastoreMofromVM(VmwareHypervisorHost hyperHost, VmwareContext context, - VirtualMachineMO vmMo, DiskTO disk) throws Exception { + VirtualMachineMO vmMo, DiskTO disk, VirtualMachineDiskInfoBuilder diskInfoBuilder) throws Exception { assert (hyperHost != null) && (context != null); List> diskDatastores = vmMo.getAllDiskDatastores(); - VirtualMachineDiskInfoBuilder diskInfoBuilder = vmMo.getDiskInfoBuilder(); VolumeObjectTO volume = (VolumeObjectTO) disk.getData(); String diskBackingFileBaseName = volume.getPath(); for (Pair diskDatastore : diskDatastores) { diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index 3fdf534b7f4..178873cac83 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -2055,16 +2055,29 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic _volsDao.detachVolume(volume.getId()); String datastoreName = answer.getContextParam("datastoreName"); if (datastoreName != null) { - List storagePoolVOS = _storagePoolDao.listPoolsByLikePath(datastoreName); - if (storagePoolVOS != null && !storagePoolVOS.isEmpty()) { + StoragePoolVO storagePoolVO = _storagePoolDao.findByUuid(datastoreName); + if (storagePoolVO != null) { VolumeVO volumeVO = _volsDao.findById(volumeId); - volumeVO.setPoolId(storagePoolVOS.get(0).getId()); + volumeVO.setPoolId(storagePoolVO.getId()); _volsDao.update(volumeVO.getId(), volumeVO); } else { s_logger.warn(String.format("Unable to find datastore %s while updating the new datastore of the volume %d", datastoreName, volumeId)); } } + String volumePath = answer.getContextParam("volumePath"); + if (volumePath != null) { + VolumeVO volumeVO = _volsDao.findById(volumeId); + volumeVO.setPath(volumePath); + _volsDao.update(volumeVO.getId(), volumeVO); + } + + String chainInfo = answer.getContextParam("chainInfo"); + if (chainInfo != null) { + VolumeVO volumeVO = _volsDao.findById(volumeId); + volumeVO.setChainInfo(chainInfo); + _volsDao.update(volumeVO.getId(), volumeVO); + } // volume.getPoolId() should be null if the VM we are detaching the disk from has never been started before if (volume.getPoolId() != null) { DataStore dataStore = dataStoreMgr.getDataStore(volume.getPoolId(), DataStoreRole.Primary); diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 8395467039d..454ec7edc30 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -2896,14 +2896,20 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir } s_logger.debug("Found no ongoing snapshots on volume of type ROOT, for the vm with id " + vmId); - List volumes = getVolumesFromIds(cmd); + List volumesToBeDeleted = getVolumesFromIds(cmd); - checkForUnattachedVolumes(vmId, volumes); - validateVolumes(volumes); + checkForUnattachedVolumes(vmId, volumesToBeDeleted); + validateVolumes(volumesToBeDeleted); stopVirtualMachine(vmId, VmDestroyForcestop.value()); - detachVolumesFromVm(volumes); + if (vm.getHypervisorType() == HypervisorType.VMware) { + List allVolumes = _volsDao.findByInstance(vm.getId()); + allVolumes.removeIf(vol -> vol.getVolumeType() == Volume.Type.ROOT); + detachVolumesFromVm(allVolumes); + } else { + detachVolumesFromVm(volumesToBeDeleted); + } UserVm destroyedVm = destroyVm(vmId, expunge); if (expunge) { @@ -2912,7 +2918,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir } } - deleteVolumesFromVm(volumes); + deleteVolumesFromVm(volumesToBeDeleted); return destroyedVm; } From 388a9c2c6daf40a581658e091ed6f01a9120a619 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Tue, 6 Oct 2020 11:31:25 +0530 Subject: [PATCH 121/164] Reconcile chaininfo after attaching volume --- .../cloud/storage/resource/VmwareStorageProcessor.java | 8 ++++++-- .../main/java/com/cloud/storage/VolumeApiServiceImpl.java | 6 ++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java index 7d5b0ba839d..96a22f11caa 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -2184,6 +2184,9 @@ public class VmwareStorageProcessor implements StorageProcessor { } vmMo.attachDisk(new String[] { datastoreVolumePath }, morDs, diskController); + VirtualMachineDiskInfoBuilder diskInfoBuilder = vmMo.getDiskInfoBuilder(); + VirtualMachineDiskInfo diskInfo = diskInfoBuilder.getDiskInfoByBackingFileBaseName(volumePath, dsMo.getName()); + chainInfo = _gson.toJson(diskInfo); if (isManaged) { expandVirtualDisk(vmMo, datastoreVolumePath, volumeTO.getSize()); @@ -2201,15 +2204,16 @@ public class VmwareStorageProcessor implements StorageProcessor { } if (datastoreChangeObserved) { answer.setContextParam("datastoreName", dsMo.getCustomFieldValue(CustomFieldConstants.CLOUD_UUID)); - answer.setContextParam("chainInfo", chainInfo); } if (volumePathChangeObserved) { answer.setContextParam("volumePath", volumePath); - answer.setContextParam("chainInfo", chainInfo); } } + if (chainInfo != null && !chainInfo.isEmpty()) + answer.setContextParam("chainInfo", chainInfo); + return answer; } catch (Throwable e) { if (e instanceof RemoteException) { diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index 178873cac83..9242052ae58 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -3109,7 +3109,13 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic if (volumeToAttachStoragePool.isManaged() && volumeToAttach.getPath() == null) { volumeToAttach.setPath(answer.getDisk().getPath()); + _volsDao.update(volumeToAttach.getId(), volumeToAttach); + } + String chainInfo = answer.getContextParam("chainInfo"); + if (chainInfo != null) { + volumeToAttach = _volsDao.findById(volumeToAttach.getId()); + volumeToAttach.setChainInfo(chainInfo); _volsDao.update(volumeToAttach.getId(), volumeToAttach); } } else { From 9dd1d60db978806b84f021e841a09edb8cfbdf80 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Wed, 7 Oct 2020 11:35:26 +0530 Subject: [PATCH 122/164] Fix multidisk VM deployment on vVol datastore --- .../resource/VmwareStorageProcessor.java | 34 ++++++++++++------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java index 96a22f11caa..bc2639235a9 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -849,8 +849,8 @@ public class VmwareStorageProcessor implements StorageProcessor { s_logger.warn("Template host in vSphere is not in connected state, request template reload"); return new CopyCmdAnswer("Template host in vSphere is not in connected state, request template reload"); } - if (dsMo.getDatastoreType().equalsIgnoreCase("VVOL") && volume.getVolumeType() == Volume.Type.ROOT) { - vmdkFileBaseName = cloneVMwithVMname(context, hyperHost, template, vmTemplate, volume, dcMo, dsMo); + if (dsMo.getDatastoreType().equalsIgnoreCase("VVOL")) { + vmdkFileBaseName = cloneVMforVvols(context, hyperHost, template, vmTemplate, volume, dcMo, dsMo); } else { vmdkFileBaseName = createVMFolderWithVMName(context, hyperHost, template, vmTemplate, volume, dcMo, dsMo, searchExcludedFolders); } @@ -858,9 +858,11 @@ public class VmwareStorageProcessor implements StorageProcessor { // restoreVM - move the new ROOT disk into corresponding VM folder VirtualMachineMO restoreVmMo = dcMo.findVm(volume.getVmName()); if (restoreVmMo != null) { - String vmNameInVcenter = restoreVmMo.getName(); // VM folder name in datastore will be VM's name in vCenter. - if (dsMo.folderExists(String.format("[%s]", dsMo.getName()), vmNameInVcenter)) { - VmwareStorageLayoutHelper.syncVolumeToVmDefaultFolder(dcMo, vmNameInVcenter, dsMo, vmdkFileBaseName, searchExcludedFolders); + if (!dsMo.getDatastoreType().equalsIgnoreCase("VVOL")) { + String vmNameInVcenter = restoreVmMo.getName(); // VM folder name in datastore will be VM's name in vCenter. + if (dsMo.folderExists(String.format("[%s]", dsMo.getName()), vmNameInVcenter)) { + VmwareStorageLayoutHelper.syncVolumeToVmDefaultFolder(dcMo, vmNameInVcenter, dsMo, vmdkFileBaseName, searchExcludedFolders); + } } } } @@ -887,24 +889,32 @@ public class VmwareStorageProcessor implements StorageProcessor { } } - private String cloneVMwithVMname(VmwareContext context, VmwareHypervisorHost hyperHost, TemplateObjectTO template, - VirtualMachineMO vmTemplate, VolumeObjectTO volume, DatacenterMO dcMo, DatastoreMO dsMo) throws Exception { + private String cloneVMforVvols(VmwareContext context, VmwareHypervisorHost hyperHost, TemplateObjectTO template, + VirtualMachineMO vmTemplate, VolumeObjectTO volume, DatacenterMO dcMo, DatastoreMO dsMo) throws Exception { ManagedObjectReference morDatastore = dsMo.getMor(); ManagedObjectReference morPool = hyperHost.getHyperHostOwnerResourcePool(); ManagedObjectReference morCluster = hyperHost.getHyperHostCluster(); if (template.getSize() != null) { _fullCloneFlag = volume.getSize() > template.getSize() ? true : _fullCloneFlag; } + String vmName = volume.getVmName(); + if (volume.getVolumeType() == Volume.Type.DATADISK) + vmName = volume.getName(); if (!_fullCloneFlag) { - createVMLinkedClone(vmTemplate, dcMo, volume.getVmName(), morDatastore, morPool); + createVMLinkedClone(vmTemplate, dcMo, vmName, morDatastore, morPool); } else { - createVMFullClone(vmTemplate, dcMo, dsMo, volume.getVmName(), morDatastore, morPool); + createVMFullClone(vmTemplate, dcMo, dsMo, vmName, morDatastore, morPool); } - VirtualMachineMO vmMo = new ClusterMO(context, morCluster).findVmOnHyperHost(volume.getVmName()); + VirtualMachineMO vmMo = new ClusterMO(context, morCluster).findVmOnHyperHost(vmName); assert (vmMo != null); - - return vmMo.getVmdkFileBaseNames().get(0); + String vmdkFileBaseName = vmMo.getVmdkFileBaseNames().get(0); + if (volume.getVolumeType() == Volume.Type.DATADISK) { + s_logger.info("detach disks from volume-wrapper VM " + vmName); + vmMo.detachAllDisks(); + vmMo.destroy(); + } + return vmdkFileBaseName; } private String createVMFolderWithVMName(VmwareContext context, VmwareHypervisorHost hyperHost, TemplateObjectTO template, From db46a7ab210010a16f6b8d6f5e46a5e960e086c3 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Wed, 7 Oct 2020 11:43:58 +0530 Subject: [PATCH 123/164] Fix migrate root volume issue caused by regression from one of the commits --- .../java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java index 2135e2b1816..01bbf02a7b0 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java @@ -2641,7 +2641,7 @@ public class VirtualMachineMO extends BaseMO { VirtualDeviceBackingInfo backingInfo = ((VirtualDisk)device).getBacking(); if (backingInfo instanceof VirtualDiskFlatVer2BackingInfo) { VirtualDiskFlatVer2BackingInfo diskBackingInfo = (VirtualDiskFlatVer2BackingInfo)backingInfo; - disks.add(new Pair(new Integer(device.getUnitNumber()), diskBackingInfo.getDatastore())); + disks.add(new Pair(new Integer(device.getKey()), diskBackingInfo.getDatastore())); } } } From d69283a01337caf6b67a9409e9a673c7c17ba5b8 Mon Sep 17 00:00:00 2001 From: nvazquez Date: Wed, 7 Oct 2020 10:25:49 -0300 Subject: [PATCH 124/164] Fix reset/restore VM --- .../cloud/agent/api/to/DeployAsIsInfoTO.java | 8 ++- .../com/cloud/vm/VirtualMachineProfile.java | 1 + .../hypervisor/guru/VmwareVmImplementer.java | 3 +- .../vmware/resource/VmwareResource.java | 67 +++++++++++++++++-- .../java/com/cloud/vm/UserVmManagerImpl.java | 8 ++- 5 files changed, 77 insertions(+), 10 deletions(-) diff --git a/api/src/main/java/com/cloud/agent/api/to/DeployAsIsInfoTO.java b/api/src/main/java/com/cloud/agent/api/to/DeployAsIsInfoTO.java index ce78f83fb4d..1c5bf999d84 100644 --- a/api/src/main/java/com/cloud/agent/api/to/DeployAsIsInfoTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/DeployAsIsInfoTO.java @@ -31,16 +31,18 @@ public class DeployAsIsInfoTO { @LogLevel(LogLevel.Log4jLevel.Off) private Map properties = new HashMap<>(); private Map nicAdapterMap = new HashMap(); + boolean replaceVm; public DeployAsIsInfoTO() { } public DeployAsIsInfoTO(String templatePath, String destStoragePool, Map properties, - Map nicAdapterMap) { + Map nicAdapterMap, boolean replaceVm) { this.templatePath = templatePath; this.destStoragePool = destStoragePool; this.properties = properties; this.nicAdapterMap = nicAdapterMap; + this.replaceVm = replaceVm; } public String getTemplatePath() { @@ -58,4 +60,8 @@ public class DeployAsIsInfoTO { public Map getNicAdapterMap() { return nicAdapterMap; } + + public boolean isReplaceVm() { + return replaceVm; + } } diff --git a/api/src/main/java/com/cloud/vm/VirtualMachineProfile.java b/api/src/main/java/com/cloud/vm/VirtualMachineProfile.java index c17a716666d..be90b12e91a 100644 --- a/api/src/main/java/com/cloud/vm/VirtualMachineProfile.java +++ b/api/src/main/java/com/cloud/vm/VirtualMachineProfile.java @@ -69,6 +69,7 @@ public interface VirtualMachineProfile { public static final Param BootType = new Param("BootType"); public static final Param BootIntoSetup = new Param("enterHardwareSetup"); public static final Param PreserveNics = new Param("PreserveNics"); + public static final Param ReplaceDeployAsIs = new Param("ReplaceDeployAsIs"); private String name; diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java index c12c5c20e2d..c5a01979105 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java @@ -195,7 +195,8 @@ class VmwareVmImplementer { templatePath = deployAsIsHelper.getAllocatedVirtualMachineTemplatePath(vm, configuration, destStoragePool); } Map nicsAdapterMapping = deployAsIsHelper.getAllocatedVirtualMachineNicsAdapterMapping(vm, to.getNics()); - DeployAsIsInfoTO info = new DeployAsIsInfoTO(templatePath, destStoragePool, properties, nicsAdapterMapping); + boolean replaceVm = vm.getParameter(VirtualMachineProfile.Param.ReplaceDeployAsIs) != null ? (Boolean) vm.getParameter(VirtualMachineProfile.Param.ReplaceDeployAsIs) : false; + DeployAsIsInfoTO info = new DeployAsIsInfoTO(templatePath, destStoragePool, properties, nicsAdapterMapping, replaceVm); to.setDeployAsIsInfo(info); } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index d2b082ae393..4a810c664e7 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -48,6 +48,7 @@ import javax.xml.datatype.XMLGregorianCalendar; import com.cloud.agent.api.to.DataTO; import com.cloud.agent.api.to.DeployAsIsInfoTO; import com.cloud.agent.api.ValidateVcenterDetailsCommand; +import com.cloud.storage.resource.VmwareStorageLayoutType; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.storage.configdrive.ConfigDrive; import org.apache.cloudstack.storage.to.TemplateObjectTO; @@ -1736,6 +1737,55 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } } + private void restoreVirtualMachineVolumesFromTemplate(String vmInternalCSName, VirtualMachineTO vmSpec, + VmwareHypervisorHost host, VirtualMachineMO virtualMachineMO, + VmwareContext context, DatacenterMO dcMo) throws Exception { + DeployAsIsInfoTO deployAsIsInfo = vmSpec.getDeployAsIsInfo(); + if (s_logger.isInfoEnabled()) { + s_logger.info("Restoring VM " + vmInternalCSName + " volumes from template as-is"); + } + String deployAsIsTemplate = deployAsIsInfo.getTemplatePath(); + String destDatastore = deployAsIsInfo.getDestStoragePool(); + + String auxVMName = vmInternalCSName + "-aux"; + _storageProcessor.cloneVMFromTemplate(host, deployAsIsTemplate, auxVMName, destDatastore); + VirtualMachineMO auxVM = host.findVmOnHyperHost(auxVMName); + if (auxVM == null) { + s_logger.info("Cloned deploy-as-is VM " + auxVMName + " is not in this host, relocating it"); + auxVM = takeVmFromOtherHyperHost(host, auxVMName); + } + ManagedObjectReference morDatastore = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(host, destDatastore); + DatastoreMO dsMo = new DatastoreMO(context, morDatastore); + List vmdkFileBaseNames = auxVM.getVmdkFileBaseNames(); + for (String vmdkFileBaseName : vmdkFileBaseNames) { + s_logger.info("Move volume out of volume-wrapper VM " + vmdkFileBaseName); + String[] vmwareLayoutFilePair = VmwareStorageLayoutHelper.getVmdkFilePairDatastorePath(dsMo, auxVMName, vmdkFileBaseName, VmwareStorageLayoutType.VMWARE, !_fullCloneFlag); + String[] legacyCloudStackLayoutFilePair = VmwareStorageLayoutHelper.getVmdkFilePairDatastorePath(dsMo, auxVMName, vmdkFileBaseName, VmwareStorageLayoutType.CLOUDSTACK_LEGACY, !_fullCloneFlag); + for (int i=0; i[]{VirtualDisk.class}); + for (String vmdkFileBaseName : vmdkFileBaseNames) { + String newPath = null; + if (dsMo.folderExists(String.format("[%s]", dsMo.getName()), vmNameInVcenter)) { + newPath = VmwareStorageLayoutHelper.syncVolumeToVmDefaultFolder(dcMo, vmNameInVcenter, dsMo, vmdkFileBaseName, null); + } + virtualMachineMO.attachDisk(new String[] {newPath}, morDatastore); + } + } + + private boolean isRestoreParameterSet(VirtualMachineTO vmSpec) { + DeployAsIsInfoTO deployAsIsInfo = vmSpec.getDeployAsIsInfo(); + if (deployAsIsInfo != null) { + return deployAsIsInfo.isReplaceVm(); + } + return false; + } + protected StartAnswer execute(StartCommand cmd) { if (s_logger.isInfoEnabled()) { s_logger.info("Executing resource StartCommand: " + getHumanReadableBytesJson(_gson.toJson(cmd))); @@ -1912,6 +1962,11 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } } + if (deployAsIs && isRestoreParameterSet(vmSpec)) { + restoreVirtualMachineVolumesFromTemplate(vmInternalCSName, vmSpec, hyperHost, vmMo, context, dcMo); + mapSpecDisksToClonedDisks(vmMo, vmInternalCSName, specDisks); + } + int disksChanges = getDisksChangesNumberFromDisksSpec(disks, deployAsIs); int totalChangeDevices = disksChanges + nics.length; if (deployAsIsInfo != null && deployAsIsInfo.getProperties() != null) { @@ -2517,13 +2572,11 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa private String getBootModeFromVmSpec(VirtualMachineTO vmSpec, boolean deployAsIs) { String bootMode = null; - if (!deployAsIs) { - if (vmSpec.getDetails().containsKey(VmDetailConstants.BOOT_MODE)) { - bootMode = vmSpec.getDetails().get(VmDetailConstants.BOOT_MODE); - } - if (null == bootMode) { - bootMode = ApiConstants.BootType.BIOS.toString(); - } + if (vmSpec.getDetails().containsKey(VmDetailConstants.BOOT_MODE)) { + bootMode = vmSpec.getDetails().get(VmDetailConstants.BOOT_MODE); + } + if (bootMode == null) { + bootMode = ApiConstants.BootType.BIOS.toString(); } return bootMode; } diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 454ec7edc30..4f78645730f 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -6860,9 +6860,15 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir if (needRestart) { try { if (vm.getDetail(VmDetailConstants.PASSWORD) != null) { - params = new HashMap(); + params = new HashMap<>(); params.put(VirtualMachineProfile.Param.VmPassword, password); } + if (template.isDeployAsIs()) { + if (params == null) { + params = new HashMap<>(); + } + params.put(VirtualMachineProfile.Param.ReplaceDeployAsIs, true); + } _itMgr.start(vm.getUuid(), params); vm = _vmDao.findById(vmId); if (template.isEnablePassword()) { From 3063b6cae5664581604849b247b75551bef2272d Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Wed, 7 Oct 2020 23:17:14 +0530 Subject: [PATCH 125/164] Fixed import VM for NFS storage which added as presetup. Previous code assumes the NFS storage is always added by CloudStack, but NFS can also be added as presetup storagepool. --- .../org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java b/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java index ecf1145dfb0..532f5f9e76c 100644 --- a/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java @@ -499,7 +499,9 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager { break; } } - } else { + } + + if (storagePool == null) { List pools = primaryDataStoreDao.listPoolsByCluster(cluster.getId()); pools.addAll(primaryDataStoreDao.listByDataCenterId(zone.getId())); for (StoragePool pool : pools) { From dc4f913afaa3def370511bc47cb8eeab2002e20f Mon Sep 17 00:00:00 2001 From: nvazquez Date: Thu, 8 Oct 2020 14:48:20 -0300 Subject: [PATCH 126/164] Fix restrictions on attach and restore VMs --- .../src/main/java/com/cloud/storage/VolumeApiServiceImpl.java | 3 ++- server/src/main/java/com/cloud/vm/UserVmManagerImpl.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index 9242052ae58..75847bde637 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -1501,8 +1501,9 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic UserVmVO vm = _userVmDao.findById(vmId); VolumeVO exstingVolumeOfVm = null; + VMTemplateVO template = _templateDao.findById(vm.getTemplateId()); List rootVolumesOfVm = _volsDao.findByInstanceAndType(vmId, Volume.Type.ROOT); - if (rootVolumesOfVm.size() > 1) { + if (rootVolumesOfVm.size() > 1 && template != null && !template.isDeployAsIs()) { throw new CloudRuntimeException("The VM " + vm.getHostName() + " has more than one ROOT volume and is in an invalid state."); } else { if (!rootVolumesOfVm.isEmpty()) { diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 4f78645730f..e1b7d2cb7c2 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -6730,13 +6730,14 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir needRestart = true; } + VMTemplateVO currentTemplate = _templateDao.findById(vm.getTemplateId()); List rootVols = _volsDao.findByInstanceAndType(vmId, Volume.Type.ROOT); if (rootVols.isEmpty()) { InvalidParameterValueException ex = new InvalidParameterValueException("Can not find root volume for VM " + vm.getUuid()); ex.addProxyObject(vm.getUuid(), "vmId"); throw ex; } - if (rootVols.size() > 1) { + if (rootVols.size() > 1 && currentTemplate != null && !currentTemplate.isDeployAsIs()) { InvalidParameterValueException ex = new InvalidParameterValueException("There are " + rootVols.size() + " root volumes for VM " + vm.getUuid()); ex.addProxyObject(vm.getUuid(), "vmId"); throw ex; From cee71c67ea3673f1994f6db0cb3153a9d9b5369c Mon Sep 17 00:00:00 2001 From: nvazquez Date: Thu, 8 Oct 2020 15:03:18 -0300 Subject: [PATCH 127/164] Remove deploy as is option from the old UI --- ui/l10n/en.js | 1 - ui/scripts/docs.js | 4 ---- ui/scripts/templates.js | 38 +++----------------------------------- 3 files changed, 3 insertions(+), 40 deletions(-) diff --git a/ui/l10n/en.js b/ui/l10n/en.js index 7e705131bcf..8e25c54b04c 100644 --- a/ui/l10n/en.js +++ b/ui/l10n/en.js @@ -674,7 +674,6 @@ var dictionary = { "label.deleting.processing":"Deleting....", "label.deny":"Deny", "label.deployment.planner":"Deployment planner", -"label.deploy.as.is":"Deploy As-Is", "label.description":"Description", "label.destination.physical.network.id":"Destination physical network ID", "label.destination.zone":"Destination Zone", diff --git a/ui/scripts/docs.js b/ui/scripts/docs.js index 19a7ad0441b..a801a6b52e0 100755 --- a/ui/scripts/docs.js +++ b/ui/scripts/docs.js @@ -1243,10 +1243,6 @@ cloudStack.docs = { desc: 'The Management Server will download the file from the specified URL, such as http://my.web.server/filename.vhd.gz', externalLink: '' }, - helpRegisterTemplateDeployAsIs: { - desc: 'Vmware Only: Deploy with specifications from OVF instead of orchestrated specs', - externalLink: '' - }, helpRegisterTemplateDirectDownload: { desc: 'KVM Only: Secondary Storage is bypassed and template/ISO is downloaded to Primary Storage on deployment', externalLink: '' diff --git a/ui/scripts/templates.js b/ui/scripts/templates.js index aeae6f04cc0..8660c5c34c9 100644 --- a/ui/scripts/templates.js +++ b/ui/scripts/templates.js @@ -245,10 +245,9 @@ args.$select.change(function() { var $form = $(this).closest('form'); if ($(this).val() == "VMware") { - $form.find('.form-item[rel=rootDiskControllerType]').css('display', 'inline-block'); - $form.find('.form-item[rel=nicAdapterType]').css('display', 'inline-block'); + $form.find('.form-item[rel=rootDiskControllerType]').hide(); + $form.find('.form-item[rel=nicAdapterType]').hide(); $form.find('.form-item[rel=keyboardType]').css('display', 'inline-block'); - $form.find('.form-item[rel=deployAsIs]').css('display', 'inline-block'); $form.find('.form-item[rel=xenserverToolsVersion61plus]').hide(); $form.find('.form-item[rel=rootDiskControllerTypeKVM]').hide(); $form.find('.form-item[rel=directdownload]').hide(); @@ -259,7 +258,6 @@ $form.find('.form-item[rel=keyboardType]').hide(); $form.find('.form-item[rel=rootDiskControllerTypeKVM]').hide(); $form.find('.form-item[rel=directdownload]').hide(); - $form.find('.form-item[rel=deployAsIs]').hide(); $form.find('.form-item[rel=requireshvm]').css('display', 'inline-block'); if (isAdmin()) { @@ -270,7 +268,6 @@ $form.find('.form-item[rel=nicAdapterType]').hide(); $form.find('.form-item[rel=keyboardType]').hide(); $form.find('.form-item[rel=xenserverToolsVersion61plus]').hide(); - $form.find('.form-item[rel=deployAsIs]').hide(); $form.find('.form-item[rel=rootDiskControllerTypeKVM]').css('display', 'inline-block'); $('#label_root_disk_controller').prop('selectedIndex', 2); $form.find('.form-item[rel=requireshvm]').css('display', 'inline-block'); @@ -284,7 +281,6 @@ $form.find('.form-item[rel=xenserverToolsVersion61plus]').hide(); $form.find('.form-item[rel=rootDiskControllerTypeKVM]').hide(); $form.find('.form-item[rel=directdownload]').hide(); - $form.find('.form-item[rel=deployAsIs]').hide(); $form.find('.form-item[rel=requireshvm]').css('display', 'inline-block'); } }); @@ -467,13 +463,6 @@ }); } }, - deployAsIs : { - label: 'label.deploy.as.is', - docID: 'helpRegisterTemplateDeployAsIs', - isBoolean: true, - dependsOn: 'hypervisor', - isHidden: true - }, // fields for hypervisor == "VMware" (ends here) format: { @@ -694,11 +683,6 @@ 'details[0].keyboard': args.data.keyboardType }); } - if (args.$form.find('.form-item[rel=deployAsIs]').css("display") != "none" && args.data.deployAsIs != "") { - $.extend(data, { - deployAsIs: (args.data.deployAsIs == "on") ? "true" : "false" - }); - } // for hypervisor == VMware (ends here) $.ajax({ @@ -1924,11 +1908,6 @@ isBoolean: true, converter: cloudStack.converters.toBooleanText }, - deployAsIs: { - label: 'label.deploy.as.is', - isBoolean: true, - converter: cloudStack.converters.toBooleanText - }, isextractable: { label: 'label.extractable.lower', isBoolean: true, @@ -2811,11 +2790,6 @@ docID: 'helpRegisterISOFeatured', isBoolean: true, isHidden: true - }, - deployAsIs : { - label: 'label.deploy.as.is', - docID: 'helpRegisterTemplateDeployAsIs', - isBoolean: true } } }, @@ -2829,8 +2803,7 @@ zoneid: args.data.zone, isextractable: (args.data.isExtractable == "on"), bootable: (args.data.isBootable == "on"), - directdownload: (args.data.directdownload == "on"), - deployAsIs: (args.data.deployAsIs == "on") + directdownload: (args.data.directdownload == "on") }; if (args.$form.find('.form-item[rel=osTypeId]').css("display") != "none") { @@ -3667,11 +3640,6 @@ isBoolean: true, converter: cloudStack.converters.toBooleanText }, - deployAsIs: { - label: 'label.deploy.as.is', - isBoolean: true, - converter: cloudStack.converters.toBooleanText - }, size: { label: 'label.size', converter: function(args) { From 07abcf570581f84c281c8fa1b72a16dd5d0cdc47 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Fri, 9 Oct 2020 02:01:56 +0530 Subject: [PATCH 128/164] During migrate volume command, when operation timed out exception or any exception is occured it is not handled properly to clean the volume_store_ref entry. Fixed it to clean the volume_store_ref entry upon on any exception --- .../motion/AncientDataMotionStrategy.java | 62 +++++++++++-------- 1 file changed, 36 insertions(+), 26 deletions(-) diff --git a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java index 7c930fb34c9..c49ffba0b82 100644 --- a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java +++ b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java @@ -356,37 +356,47 @@ public class AncientDataMotionStrategy implements DataMotionStrategy { DataObject objOnImageStore = imageStore.create(srcData); objOnImageStore.processEvent(Event.CreateOnlyRequested); - Answer answer = copyObject(srcData, objOnImageStore); - if (answer == null || !answer.getResult()) { - if (answer != null) { - s_logger.debug("copy to image store failed: " + answer.getDetails()); + Answer answer = null; + try { + answer = copyObject(srcData, objOnImageStore); + + if (answer == null || !answer.getResult()) { + if (answer != null) { + s_logger.debug("copy to image store failed: " + answer.getDetails()); + } + objOnImageStore.processEvent(Event.OperationFailed); + imageStore.delete(objOnImageStore); + return answer; } - objOnImageStore.processEvent(Event.OperationFailed); - imageStore.delete(objOnImageStore); - return answer; - } - objOnImageStore.processEvent(Event.OperationSuccessed, answer); + objOnImageStore.processEvent(Event.OperationSuccessed, answer); - objOnImageStore.processEvent(Event.CopyingRequested); + objOnImageStore.processEvent(Event.CopyingRequested); - CopyCommand cmd = new CopyCommand(objOnImageStore.getTO(), addFullCloneFlagOnVMwareDest(destData.getTO()), _copyvolumewait, VirtualMachineManager.ExecuteInSequence.value()); - EndPoint ep = selector.select(objOnImageStore, destData); - if (ep == null) { - String errMsg = "No remote endpoint to send command, check if host or ssvm is down?"; - s_logger.error(errMsg); - answer = new Answer(cmd, false, errMsg); - } else { - answer = ep.sendMessage(cmd); - } - - if (answer == null || !answer.getResult()) { - if (answer != null) { - s_logger.debug("copy to primary store failed: " + answer.getDetails()); + CopyCommand cmd = new CopyCommand(objOnImageStore.getTO(), addFullCloneFlagOnVMwareDest(destData.getTO()), _copyvolumewait, VirtualMachineManager.ExecuteInSequence.value()); + EndPoint ep = selector.select(objOnImageStore, destData); + if (ep == null) { + String errMsg = "No remote endpoint to send command, check if host or ssvm is down?"; + s_logger.error(errMsg); + answer = new Answer(cmd, false, errMsg); + } else { + answer = ep.sendMessage(cmd); } - objOnImageStore.processEvent(Event.OperationFailed); - imageStore.delete(objOnImageStore); - return answer; + + if (answer == null || !answer.getResult()) { + if (answer != null) { + s_logger.debug("copy to primary store failed: " + answer.getDetails()); + } + objOnImageStore.processEvent(Event.OperationFailed); + imageStore.delete(objOnImageStore); + return answer; + } + } catch (Exception e) { + if (imageStore.exists(objOnImageStore)) { + objOnImageStore.processEvent(Event.OperationFailed); + imageStore.delete(objOnImageStore); + } + throw e; } objOnImageStore.processEvent(Event.OperationSuccessed); From a54436b790b758800f8afe4d1c29fe278894e48b Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Fri, 9 Oct 2020 04:56:15 +0530 Subject: [PATCH 129/164] service offering response: added vsphere storage policy to response if exists --- .../api/response/ServiceOfferingResponse.java | 13 ++++++++++++ .../META-INF/db/schema-41400to41500.sql | 4 ++++ .../query/dao/ServiceOfferingJoinDaoImpl.java | 21 +++++++++++++++++-- .../api/query/vo/ServiceOfferingJoinVO.java | 8 +++++++ 4 files changed, 44 insertions(+), 2 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/response/ServiceOfferingResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/ServiceOfferingResponse.java index 9d5e9ee5d7c..e13735bd1c3 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/ServiceOfferingResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/ServiceOfferingResponse.java @@ -196,6 +196,10 @@ public class ServiceOfferingResponse extends BaseResponse { @Param(description = "the cache mode to use for this disk offering. none, writeback or writethrough", since = "4.14") private String cacheMode; + @SerializedName("vspherestoragepolicy") + @Param(description = "the vsphere storage policy tagged to the service offering in case of VMware", since = "4.15") + private String vsphereStoragePolicy; + public ServiceOfferingResponse() { } @@ -455,4 +459,13 @@ public class ServiceOfferingResponse extends BaseResponse { public void setCacheMode(String cacheMode) { this.cacheMode = cacheMode; } + + public String getVsphereStoragePolicy() { + return vsphereStoragePolicy; + } + + public void setVsphereStoragePolicy(String vsphereStoragePolicy) { + this.vsphereStoragePolicy = vsphereStoragePolicy; + } + } diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql b/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql index 788346d7064..ed94ab43efb 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql @@ -415,6 +415,7 @@ CREATE VIEW `cloud`.`service_offering_view` AS `service_offering`.`sort_key` AS `sort_key`, `service_offering`.`is_volatile` AS `is_volatile`, `service_offering`.`deployment_planner` AS `deployment_planner`, + `vsphere_storage_policy`.`value` AS `vsphere_storage_policy`, GROUP_CONCAT(DISTINCT(domain.id)) AS domain_id, GROUP_CONCAT(DISTINCT(domain.uuid)) AS domain_uuid, GROUP_CONCAT(DISTINCT(domain.name)) AS domain_name, @@ -450,6 +451,9 @@ CREATE VIEW `cloud`.`service_offering_view` AS LEFT JOIN `cloud`.`service_offering_details` AS `max_memory_details` ON `max_memory_details`.`service_offering_id` = `disk_offering`.`id` AND `max_memory_details`.`name` = 'maxmemory' + LEFT JOIN + `cloud`.`service_offering_details` AS `vsphere_storage_policy` ON `vsphere_storage_policy`.`service_offering_id` = `disk_offering`.`id` + AND `vsphere_storage_policy`.`name` = 'storagepolicy' WHERE `disk_offering`.`state`='Active' GROUP BY diff --git a/server/src/main/java/com/cloud/api/query/dao/ServiceOfferingJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/ServiceOfferingJoinDaoImpl.java index add64155375..4c025b99e9d 100644 --- a/server/src/main/java/com/cloud/api/query/dao/ServiceOfferingJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/ServiceOfferingJoinDaoImpl.java @@ -17,7 +17,11 @@ package com.cloud.api.query.dao; import java.util.List; +import java.util.Map; +import com.cloud.dc.VsphereStoragePolicyVO; +import com.cloud.dc.dao.VsphereStoragePolicyDao; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.response.ServiceOfferingResponse; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -30,10 +34,15 @@ import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; +import javax.inject.Inject; + @Component public class ServiceOfferingJoinDaoImpl extends GenericDaoBase implements ServiceOfferingJoinDao { public static final Logger s_logger = Logger.getLogger(ServiceOfferingJoinDaoImpl.class); + @Inject + VsphereStoragePolicyDao _vsphereStoragePolicyDao; + private SearchBuilder sofIdSearch; protected ServiceOfferingJoinDaoImpl() { @@ -99,11 +108,19 @@ public class ServiceOfferingJoinDaoImpl extends GenericDaoBase offeringDetails = ApiDBUtils.getResourceDetails(offering.getId(), ResourceObjectType.ServiceOffering); + offeringResponse.setDetails(offeringDetails); offeringResponse.setObjectName("serviceoffering"); offeringResponse.setIscutomized(offering.isDynamic()); offeringResponse.setCacheMode(offering.getCacheMode()); - + if (offeringDetails != null && !offeringDetails.isEmpty()) { + String vsphereStoragePolicyId = offeringDetails.get(ApiConstants.STORAGE_POLICY); + if (vsphereStoragePolicyId != null) { + VsphereStoragePolicyVO vsphereStoragePolicyVO = _vsphereStoragePolicyDao.findById(Long.parseLong(vsphereStoragePolicyId)); + if (vsphereStoragePolicyVO != null) + offeringResponse.setVsphereStoragePolicy(vsphereStoragePolicyVO.getName()); + } + } return offeringResponse; } diff --git a/server/src/main/java/com/cloud/api/query/vo/ServiceOfferingJoinVO.java b/server/src/main/java/com/cloud/api/query/vo/ServiceOfferingJoinVO.java index a3e1cb14723..e025da5de3f 100644 --- a/server/src/main/java/com/cloud/api/query/vo/ServiceOfferingJoinVO.java +++ b/server/src/main/java/com/cloud/api/query/vo/ServiceOfferingJoinVO.java @@ -187,6 +187,9 @@ public class ServiceOfferingJoinVO extends BaseViewVO implements InternalIdentit @Column(name = "max_memory") Integer maxMemory; + @Column(name = "vsphere_storage_policy") + String vsphereStoragePolicy; + public ServiceOfferingJoinVO() { } @@ -384,4 +387,9 @@ public class ServiceOfferingJoinVO extends BaseViewVO implements InternalIdentit public Integer getMaxMemory() { return maxMemory; } + + public String getVsphereStoragePolicy() { + return vsphereStoragePolicy; + } + } From 1e4e5cf4c03ece9e02097654d414563c7c91e499 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Fri, 9 Oct 2020 05:23:11 +0530 Subject: [PATCH 130/164] Disk offering response: added vsphere storage policy to response if exists --- .../api/response/DiskOfferingResponse.java | 12 ++++ .../META-INF/db/schema-41400to41500.sql | 63 +++++++++++++++++++ .../query/dao/DiskOfferingJoinDaoImpl.java | 20 ++++++ .../api/query/vo/DiskOfferingJoinVO.java | 7 +++ 4 files changed, 102 insertions(+) diff --git a/api/src/main/java/org/apache/cloudstack/api/response/DiskOfferingResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/DiskOfferingResponse.java index 8c0fd70ff98..5398dd32bd6 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/DiskOfferingResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/DiskOfferingResponse.java @@ -151,6 +151,10 @@ public class DiskOfferingResponse extends BaseResponse { @Param(description = "whether to display the offering to the end user or not.") private Boolean displayOffering; + @SerializedName("vspherestoragepolicy") + @Param(description = "the vsphere storage policy tagged to the disk offering in case of VMware", since = "4.15") + private String vsphereStoragePolicy; + public Boolean getDisplayOffering() { return displayOffering; } @@ -351,4 +355,12 @@ public class DiskOfferingResponse extends BaseResponse { public void setIopsWriteRateMaxLength(Long iopsWriteRateMaxLength) { this.iopsWriteRateMaxLength = iopsWriteRateMaxLength; } + + public String getVsphereStoragePolicy() { + return vsphereStoragePolicy; + } + + public void setVsphereStoragePolicy(String vsphereStoragePolicy) { + this.vsphereStoragePolicy = vsphereStoragePolicy; + } } diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql b/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql index ed94ab43efb..4df87b8cb33 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql @@ -459,6 +459,69 @@ CREATE VIEW `cloud`.`service_offering_view` AS GROUP BY `service_offering`.`id`; +DROP VIEW IF EXISTS `cloud`.`disk_offering_view`; +CREATE VIEW `cloud`.`disk_offering_view` AS + SELECT + `disk_offering`.`id` AS `id`, + `disk_offering`.`uuid` AS `uuid`, + `disk_offering`.`name` AS `name`, + `disk_offering`.`display_text` AS `display_text`, + `disk_offering`.`provisioning_type` AS `provisioning_type`, + `disk_offering`.`disk_size` AS `disk_size`, + `disk_offering`.`min_iops` AS `min_iops`, + `disk_offering`.`max_iops` AS `max_iops`, + `disk_offering`.`created` AS `created`, + `disk_offering`.`tags` AS `tags`, + `disk_offering`.`customized` AS `customized`, + `disk_offering`.`customized_iops` AS `customized_iops`, + `disk_offering`.`removed` AS `removed`, + `disk_offering`.`use_local_storage` AS `use_local_storage`, + `disk_offering`.`system_use` AS `system_use`, + `disk_offering`.`hv_ss_reserve` AS `hv_ss_reserve`, + `disk_offering`.`bytes_read_rate` AS `bytes_read_rate`, + `disk_offering`.`bytes_read_rate_max` AS `bytes_read_rate_max`, + `disk_offering`.`bytes_read_rate_max_length` AS `bytes_read_rate_max_length`, + `disk_offering`.`bytes_write_rate` AS `bytes_write_rate`, + `disk_offering`.`bytes_write_rate_max` AS `bytes_write_rate_max`, + `disk_offering`.`bytes_write_rate_max_length` AS `bytes_write_rate_max_length`, + `disk_offering`.`iops_read_rate` AS `iops_read_rate`, + `disk_offering`.`iops_read_rate_max` AS `iops_read_rate_max`, + `disk_offering`.`iops_read_rate_max_length` AS `iops_read_rate_max_length`, + `disk_offering`.`iops_write_rate` AS `iops_write_rate`, + `disk_offering`.`iops_write_rate_max` AS `iops_write_rate_max`, + `disk_offering`.`iops_write_rate_max_length` AS `iops_write_rate_max_length`, + `disk_offering`.`cache_mode` AS `cache_mode`, + `disk_offering`.`sort_key` AS `sort_key`, + `disk_offering`.`type` AS `type`, + `disk_offering`.`display_offering` AS `display_offering`, + `disk_offering`.`state` AS `state`, + `vsphere_storage_policy`.`value` AS `vsphere_storage_policy`, + GROUP_CONCAT(DISTINCT(domain.id)) AS domain_id, + GROUP_CONCAT(DISTINCT(domain.uuid)) AS domain_uuid, + GROUP_CONCAT(DISTINCT(domain.name)) AS domain_name, + GROUP_CONCAT(DISTINCT(domain.path)) AS domain_path, + GROUP_CONCAT(DISTINCT(zone.id)) AS zone_id, + GROUP_CONCAT(DISTINCT(zone.uuid)) AS zone_uuid, + GROUP_CONCAT(DISTINCT(zone.name)) AS zone_name + FROM + `cloud`.`disk_offering` + LEFT JOIN + `cloud`.`disk_offering_details` AS `domain_details` ON `domain_details`.`offering_id` = `disk_offering`.`id` AND `domain_details`.`name`='domainid' + LEFT JOIN + `cloud`.`domain` AS `domain` ON FIND_IN_SET(`domain`.`id`, `domain_details`.`value`) + LEFT JOIN + `cloud`.`disk_offering_details` AS `zone_details` ON `zone_details`.`offering_id` = `disk_offering`.`id` AND `zone_details`.`name`='zoneid' + LEFT JOIN + `cloud`.`data_center` AS `zone` ON FIND_IN_SET(`zone`.`id`, `zone_details`.`value`) + LEFT JOIN + `cloud`.`disk_offering_details` AS `vsphere_storage_policy` ON `vsphere_storage_policy`.`offering_id` = `disk_offering`.`id` + AND `vsphere_storage_policy`.`name` = 'storagepolicy' + + WHERE + `disk_offering`.`state`='Active' + GROUP BY + `disk_offering`.`id`; + ALTER TABLE `cloud`.`template_spool_ref` DROP FOREIGN KEY `fk_template_spool_ref__template_id`; diff --git a/server/src/main/java/com/cloud/api/query/dao/DiskOfferingJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/DiskOfferingJoinDaoImpl.java index 9220974897c..a62fbcdce20 100644 --- a/server/src/main/java/com/cloud/api/query/dao/DiskOfferingJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/DiskOfferingJoinDaoImpl.java @@ -17,7 +17,13 @@ package com.cloud.api.query.dao; import java.util.List; +import java.util.Map; +import com.cloud.api.ApiDBUtils; +import com.cloud.dc.VsphereStoragePolicyVO; +import com.cloud.dc.dao.VsphereStoragePolicyDao; +import com.cloud.server.ResourceTag; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.response.DiskOfferingResponse; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -30,10 +36,15 @@ import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; +import javax.inject.Inject; + @Component public class DiskOfferingJoinDaoImpl extends GenericDaoBase implements DiskOfferingJoinDao { public static final Logger s_logger = Logger.getLogger(DiskOfferingJoinDaoImpl.class); + @Inject + VsphereStoragePolicyDao _vsphereStoragePolicyDao; + private final SearchBuilder dofIdSearch; private final Attribute _typeAttr; @@ -108,6 +119,15 @@ public class DiskOfferingJoinDaoImpl extends GenericDaoBase offeringDetails = ApiDBUtils.getResourceDetails(offering.getId(), ResourceTag.ResourceObjectType.DiskOffering); + if (offeringDetails != null && !offeringDetails.isEmpty()) { + String vsphereStoragePolicyId = offeringDetails.get(ApiConstants.STORAGE_POLICY); + if (vsphereStoragePolicyId != null) { + VsphereStoragePolicyVO vsphereStoragePolicyVO = _vsphereStoragePolicyDao.findById(Long.parseLong(vsphereStoragePolicyId)); + if (vsphereStoragePolicyVO != null) + diskOfferingResponse.setVsphereStoragePolicy(vsphereStoragePolicyVO.getName()); + } + } return diskOfferingResponse; } diff --git a/server/src/main/java/com/cloud/api/query/vo/DiskOfferingJoinVO.java b/server/src/main/java/com/cloud/api/query/vo/DiskOfferingJoinVO.java index 50559c7fd93..707c2a3e3b0 100644 --- a/server/src/main/java/com/cloud/api/query/vo/DiskOfferingJoinVO.java +++ b/server/src/main/java/com/cloud/api/query/vo/DiskOfferingJoinVO.java @@ -159,6 +159,9 @@ public class DiskOfferingJoinVO extends BaseViewVO implements InternalIdentity, @Column(name = "state") DiskOffering.State state; + @Column(name = "vsphere_storage_policy") + String vsphereStoragePolicy; + public DiskOfferingJoinVO() { } @@ -343,4 +346,8 @@ public class DiskOfferingJoinVO extends BaseViewVO implements InternalIdentity, public void setState(DiskOffering.State state) { this.state = state; } + + public String getVsphereStoragePolicy() { + return vsphereStoragePolicy; + } } From 46b5322d9bb6631e3136f18f005624419c21a8c0 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Sun, 11 Oct 2020 22:47:47 +0530 Subject: [PATCH 131/164] Adding vSphere storage policy to disk on start command and attach volume command --- .../cloudstack/storage/to/VolumeObjectTO.java | 9 +++++ .../subsystem/api/storage/VolumeInfo.java | 2 + .../storage/volume/VolumeObject.java | 37 +++++++++++++++++++ .../vmware/resource/VmwareResource.java | 27 +++++++++++++- .../resource/VmwareStorageProcessor.java | 3 +- .../vmware/mo/PbmProfileManagerMO.java | 10 +++++ .../vmware/mo/VirtualMachineMO.java | 14 +++++-- 7 files changed, 97 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java b/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java index f7e17f7bc14..6cd27b16cc1 100644 --- a/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java +++ b/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java @@ -64,6 +64,7 @@ public class VolumeObjectTO implements DataTO { private boolean directDownload; private boolean deployAsIs; private String updatedDataStoreUUID; + private String vSphereStoragePolicyId; public VolumeObjectTO() { @@ -105,6 +106,7 @@ public class VolumeObjectTO implements DataTO { this.migrationOptions = volume.getMigrationOptions(); this.directDownload = volume.isDirectDownload(); this.deployAsIs = volume.isDeployAsIs(); + this.vSphereStoragePolicyId = volume.getvSphereStoragePolicyId(); } public String getUuid() { @@ -329,4 +331,11 @@ public class VolumeObjectTO implements DataTO { this.updatedDataStoreUUID = updatedDataStoreUUID; } + public String getvSphereStoragePolicyId() { + return vSphereStoragePolicyId; + } + + public void setvSphereStoragePolicyId(String vSphereStoragePolicyId) { + this.vSphereStoragePolicyId = vSphereStoragePolicyId; + } } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeInfo.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeInfo.java index 06bda51a092..b1381228265 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeInfo.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeInfo.java @@ -87,4 +87,6 @@ public interface VolumeInfo extends DataObject, Volume { boolean isDeployAsIs(); String getDeployAsIsConfiguration(); + + public String getvSphereStoragePolicyId(); } diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java index 45509c51c43..6d76aba9051 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java @@ -20,12 +20,18 @@ import java.util.Date; import javax.inject.Inject; +import com.cloud.dc.VsphereStoragePolicyVO; +import com.cloud.dc.dao.VsphereStoragePolicyDao; +import com.cloud.service.dao.ServiceOfferingDetailsDao; import com.cloud.storage.MigrationOptions; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.VolumeDetailVO; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VolumeDetailsDao; import com.cloud.vm.VmDetailConstants; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.resourcedetail.dao.DiskOfferingDetailsDao; +import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import org.apache.cloudstack.engine.subsystem.api.storage.DataObjectInStore; @@ -81,9 +87,17 @@ public class VolumeObject implements VolumeInfo { VMTemplateDao templateDao; @Inject VolumeDetailsDao volumeDetailsDao; + @Inject + ServiceOfferingDetailsDao serviceOfferingDetailsDao; + @Inject + DiskOfferingDetailsDao diskOfferingDetailsDao; + @Inject + VsphereStoragePolicyDao vsphereStoragePolicyDao; + private Object payload; private MigrationOptions migrationOptions; private boolean directDownload; + private String vSphereStoragePolicyId; public VolumeObject() { _volStateMachine = Volume.State.getStateMachine(); @@ -780,6 +794,29 @@ public class VolumeObject implements VolumeInfo { } + public String getvSphereStoragePolicyId() { + if (StringUtils.isEmpty(vSphereStoragePolicyId)) { + if (Volume.Type.ROOT == getVolumeType()) { + Long vmId = volumeVO.getInstanceId(); + if (vmId != null) { + VMInstanceVO vm = vmInstanceDao.findByIdIncludingRemoved(vmId); + String storagePolicyVOid = serviceOfferingDetailsDao.getDetail(vm.getServiceOfferingId(), + ApiConstants.STORAGE_POLICY); + VsphereStoragePolicyVO vsphereStoragePolicyVO = vsphereStoragePolicyDao.findById(Long.parseLong(storagePolicyVOid)); + vSphereStoragePolicyId = vsphereStoragePolicyVO.getPolicyId(); + + } + } else { + String storagePolicyVOid = diskOfferingDetailsDao.getDetail(volumeVO.getDiskOfferingId(), + ApiConstants.STORAGE_POLICY); + VsphereStoragePolicyVO vsphereStoragePolicyVO = vsphereStoragePolicyDao.findById(Long.parseLong(storagePolicyVOid)); + vSphereStoragePolicyId = vsphereStoragePolicyVO.getPolicyId(); + + } + } + return vSphereStoragePolicyId; + } + @Override public ImageFormat getFormat() { return volumeVO.getFormat(); diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 4a810c664e7..3e515d7b5cd 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -219,6 +219,7 @@ import com.cloud.hypervisor.vmware.mo.HostMO; import com.cloud.hypervisor.vmware.mo.HostStorageSystemMO; import com.cloud.hypervisor.vmware.mo.HypervisorHostHelper; import com.cloud.hypervisor.vmware.mo.NetworkDetails; +import com.cloud.hypervisor.vmware.mo.PbmProfileManagerMO; import com.cloud.hypervisor.vmware.mo.TaskMO; import com.cloud.hypervisor.vmware.mo.StoragepodMO; import com.cloud.hypervisor.vmware.mo.VirtualEthernetCardType; @@ -289,6 +290,7 @@ import com.vmware.vim25.HostInternetScsiHba; import com.vmware.vim25.HostPortGroupSpec; import com.vmware.vim25.ManagedObjectReference; import com.vmware.vim25.NasDatastoreInfo; +import com.vmware.vim25.VirtualMachineDefinedProfileSpec; import com.vmware.vim25.ObjectContent; import com.vmware.vim25.OptionValue; import com.vmware.vim25.PerfCounterInfo; @@ -1798,6 +1800,11 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa VirtualMachineFileInfo existingVmFileInfo = null; VirtualMachineFileLayoutEx existingVmFileLayout = null; List existingDatastores = new ArrayList(); + String diskStoragePolicyId = null; + String vmStoragePolicyId = null; + VirtualMachineDefinedProfileSpec diskProfileSpec = null; + VirtualMachineDefinedProfileSpec vmProfileSpec = null; + DeployAsIsInfoTO deployAsIsInfo = vmSpec.getDeployAsIsInfo(); boolean deployAsIs = deployAsIsInfo != null; @@ -2218,8 +2225,20 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa VirtualDevice device = VmwareHelper.prepareDiskDevice(vmMo, null, controllerKey, diskChain, volumeDsDetails.first(), deviceNumber, i + 1); - if (vol.getType() == Volume.Type.ROOT) + diskStoragePolicyId = volumeTO.getvSphereStoragePolicyId(); + if (!StringUtils.isEmpty(diskStoragePolicyId)) { + PbmProfileManagerMO profMgrMo = new PbmProfileManagerMO(context); + diskProfileSpec = profMgrMo.getProfileSpec(diskStoragePolicyId); + deviceConfigSpecArray[i].getProfile().add(diskProfileSpec); + if (s_logger.isDebugEnabled()) { + s_logger.debug(String.format("Adding vSphere storage profile: %s to virtual disk [%s]", diskStoragePolicyId, _gson.toJson(device))); + } + } + if (vol.getType() == Volume.Type.ROOT) { rootDiskTO = vol; + vmStoragePolicyId = diskStoragePolicyId; + vmProfileSpec = diskProfileSpec; + } deviceConfigSpecArray[i].setDevice(device); deviceConfigSpecArray[i].setOperation(VirtualDeviceConfigSpecOperation.ADD); @@ -2393,6 +2412,12 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa setBootOptions(vmSpec, bootMode, vmConfigSpec); } + if (!StringUtils.isEmpty(vmStoragePolicyId)) { + vmConfigSpec.getVmProfile().add(vmProfileSpec); + if (s_logger.isTraceEnabled()) { + s_logger.trace(String.format("Configuring the VM %s with storage policy: %s", vmInternalCSName, vmStoragePolicyId)); + } + } // // Configure VM // diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java index bc2639235a9..cf7d933e5c7 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -2052,6 +2052,7 @@ public class VmwareStorageProcessor implements StorageProcessor { VolumeObjectTO volumeTO = (VolumeObjectTO)disk.getData(); DataStoreTO primaryStore = volumeTO.getDataStore(); String volumePath = volumeTO.getPath(); + String storagePolicyId = volumeTO.getvSphereStoragePolicyId(); String vmdkPath = isManaged ? resource.getVmdkPath(volumePath) : null; @@ -2193,7 +2194,7 @@ public class VmwareStorageProcessor implements StorageProcessor { diskController = vmMo.getRecommendedDiskController(null); } - vmMo.attachDisk(new String[] { datastoreVolumePath }, morDs, diskController); + vmMo.attachDisk(new String[] { datastoreVolumePath }, morDs, diskController, storagePolicyId); VirtualMachineDiskInfoBuilder diskInfoBuilder = vmMo.getDiskInfoBuilder(); VirtualMachineDiskInfo diskInfo = diskInfoBuilder.getDiskInfoByBackingFileBaseName(volumePath, dsMo.getName()); chainInfo = _gson.toJson(diskInfo); diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/PbmProfileManagerMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/PbmProfileManagerMO.java index a4142ecde8a..38f18d7e830 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/PbmProfileManagerMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/PbmProfileManagerMO.java @@ -27,6 +27,7 @@ import com.vmware.pbm.PbmProfileResourceType; import com.vmware.pbm.PbmProfileResourceTypeEnum; import com.vmware.vim25.ManagedObjectReference; +import com.vmware.vim25.VirtualMachineDefinedProfileSpec; import org.apache.log4j.Logger; import java.util.Collections; @@ -89,6 +90,15 @@ public class PbmProfileManagerMO extends BaseMO { resourceType.setResourceType(PbmProfileResourceTypeEnum.STORAGE.value()); return resourceType; } + + + public VirtualMachineDefinedProfileSpec getProfileSpec(String profileId) throws Exception { + VirtualMachineDefinedProfileSpec profileSpec = new VirtualMachineDefinedProfileSpec(); + PbmProfile profile = getStorageProfile(profileId); + profileSpec.setProfileId(profile.getProfileId().getUniqueId()); + return profileSpec; + } + } diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java index 01bbf02a7b0..9565ac6ff0b 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java @@ -105,6 +105,7 @@ import com.vmware.vim25.VirtualMachineSnapshotInfo; import com.vmware.vim25.VirtualMachineSnapshotTree; import com.vmware.vim25.VirtualSCSIController; import com.vmware.vim25.VirtualSCSISharing; +import com.vmware.vim25.VirtualMachineDefinedProfileSpec; import com.cloud.hypervisor.vmware.mo.SnapshotDescriptor.SnapshotInfo; import com.cloud.hypervisor.vmware.util.VmwareContext; @@ -1293,10 +1294,10 @@ public class VirtualMachineMO extends BaseMO { } public void attachDisk(String[] vmdkDatastorePathChain, ManagedObjectReference morDs) throws Exception { - attachDisk(vmdkDatastorePathChain, morDs, null); + attachDisk(vmdkDatastorePathChain, morDs, null, null); } - public void attachDisk(String[] vmdkDatastorePathChain, ManagedObjectReference morDs, String diskController) throws Exception { + public void attachDisk(String[] vmdkDatastorePathChain, ManagedObjectReference morDs, String diskController, String storagePolicyId) throws Exception { if(s_logger.isTraceEnabled()) s_logger.trace("vCenter API trace - attachDisk(). target MOR: " + _mor.getValue() + ", vmdkDatastorePath: " @@ -1337,7 +1338,14 @@ public class VirtualMachineMO extends BaseMO { deviceConfigSpec.setDevice(newDisk); deviceConfigSpec.setOperation(VirtualDeviceConfigSpecOperation.ADD); - + if (!StringUtils.isEmpty(storagePolicyId)) { + PbmProfileManagerMO profMgrMo = new PbmProfileManagerMO(getContext()); + VirtualMachineDefinedProfileSpec diskProfileSpec = profMgrMo.getProfileSpec(storagePolicyId); + deviceConfigSpec.getProfile().add(diskProfileSpec); + if (s_logger.isDebugEnabled()) { + s_logger.debug(String.format("Adding vSphere storage profile: %s to volume [%s]", storagePolicyId, vmdkDatastorePathChain[0])); + } + } reConfigSpec.getDeviceChange().add(deviceConfigSpec); ManagedObjectReference morTask = _context.getService().reconfigVMTask(_mor, reConfigSpec); From 5fdabc1cb00d40f3277147599e4a894cf68d59f9 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Mon, 12 Oct 2020 00:38:47 +0530 Subject: [PATCH 132/164] Added storage policy details to disk while creating disk and restricted migration of volumes to storage pools which are not storage policy compliance --- .../storage/volume/VolumeObject.java | 11 ++++----- .../resource/VmwareStorageProcessor.java | 7 +++--- .../com/cloud/storage/StorageManagerImpl.java | 2 +- .../cloud/storage/VolumeApiServiceImpl.java | 14 +++++++++++ .../vmware/mo/VirtualMachineMO.java | 23 ++++++++++++------- 5 files changed, 39 insertions(+), 18 deletions(-) diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java index 6d76aba9051..da759f84a7c 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java @@ -796,22 +796,21 @@ public class VolumeObject implements VolumeInfo { public String getvSphereStoragePolicyId() { if (StringUtils.isEmpty(vSphereStoragePolicyId)) { + String storagePolicyVOid = null; if (Volume.Type.ROOT == getVolumeType()) { Long vmId = volumeVO.getInstanceId(); if (vmId != null) { VMInstanceVO vm = vmInstanceDao.findByIdIncludingRemoved(vmId); - String storagePolicyVOid = serviceOfferingDetailsDao.getDetail(vm.getServiceOfferingId(), + storagePolicyVOid = serviceOfferingDetailsDao.getDetail(vm.getServiceOfferingId(), ApiConstants.STORAGE_POLICY); - VsphereStoragePolicyVO vsphereStoragePolicyVO = vsphereStoragePolicyDao.findById(Long.parseLong(storagePolicyVOid)); - vSphereStoragePolicyId = vsphereStoragePolicyVO.getPolicyId(); - } } else { - String storagePolicyVOid = diskOfferingDetailsDao.getDetail(volumeVO.getDiskOfferingId(), + storagePolicyVOid = diskOfferingDetailsDao.getDetail(volumeVO.getDiskOfferingId(), ApiConstants.STORAGE_POLICY); + } + if (storagePolicyVOid != null) { VsphereStoragePolicyVO vsphereStoragePolicyVO = vsphereStoragePolicyDao.findById(Long.parseLong(storagePolicyVOid)); vSphereStoragePolicyId = vsphereStoragePolicyVO.getPolicyId(); - } } return vSphereStoragePolicyId; diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java index cf7d933e5c7..e7349ee49ba 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -831,7 +831,7 @@ public class VmwareStorageProcessor implements StorageProcessor { synchronized (this) { s_logger.info("Delete file if exists in datastore to clear the way for creating the volume. file: " + volumeDatastorePath); VmwareStorageLayoutHelper.deleteVolumeVmdkFiles(dsMo, vmdkName, dcMo, searchExcludedFolders); - vmMo.createDisk(volumeDatastorePath, (long)(volume.getSize() / (1024L * 1024L)), morDatastore, -1); + vmMo.createDisk(volumeDatastorePath, (long)(volume.getSize() / (1024L * 1024L)), morDatastore, -1, null); vmMo.detachDisk(volumeDatastorePath, false); } } finally { @@ -2497,6 +2497,7 @@ public class VmwareStorageProcessor implements StorageProcessor { VolumeObjectTO volume = (VolumeObjectTO)cmd.getData(); DataStoreTO primaryStore = volume.getDataStore(); + String vSphereStoragePolicyId = volume.getvSphereStoragePolicyId(); try { VmwareContext context = hostService.getServiceContext(null); @@ -2534,7 +2535,7 @@ public class VmwareStorageProcessor implements StorageProcessor { synchronized (this) { try { - vmMo.createDisk(volumeDatastorePath, (int)(volume.getSize() / (1024L * 1024L)), morDatastore, vmMo.getScsiDeviceControllerKey()); + vmMo.createDisk(volumeDatastorePath, (int)(volume.getSize() / (1024L * 1024L)), morDatastore, vmMo.getScsiDeviceControllerKey(), vSphereStoragePolicyId); vmMo.detachDisk(volumeDatastorePath, false); } catch (Exception e1) { @@ -3243,7 +3244,7 @@ public class VmwareStorageProcessor implements StorageProcessor { Long volumeSizeToUse = volumeSize < dsMo.getDatastoreSummary().getFreeSpace() ? volumeSize : dsMo.getDatastoreSummary().getFreeSpace(); - vmMo.createDisk(vmdkDatastorePath, getMBsFromBytes(volumeSizeToUse), dsMo.getMor(), vmMo.getScsiDeviceControllerKey()); + vmMo.createDisk(vmdkDatastorePath, getMBsFromBytes(volumeSizeToUse), dsMo.getMor(), vmMo.getScsiDeviceControllerKey(), null); vmMo.detachDisk(vmdkDatastorePath, false); vmMo.destroy(); } diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index 212f6938e65..2ef88e44350 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -2038,7 +2038,7 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C // check cummilative result for all volumes for (Pair answer : answers) { if (!answer.second().getResult()) { - s_logger.debug(String.format("Storage pool %s is not complaince with storage policy for volume %s", pool.getName(), answer.first().getName())); + s_logger.debug(String.format("Storage pool %s is not compliance with storage policy for volume %s", pool.getUuid(), answer.first().getName())); return false; } } diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index 75847bde637..58e1bffaf37 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -2232,6 +2232,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic throw new InvalidParameterValueException("Cannot migrate volume " + vol + "to the destination storage pool " + destPool.getName() + " as the storage pool is in maintenance mode."); } + String poolUuid = destPool.getUuid(); if (destPool.getPoolType() == Storage.StoragePoolType.DatastoreCluster) { DataCenter dc = _entityMgr.findById(DataCenter.class, vol.getDataCenterId()); Pod destPoolPod = _entityMgr.findById(Pod.class, destPool.getPodId()); @@ -2277,6 +2278,19 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic updateMissingRootDiskController(vm, vol.getChainInfo()); } } + + HypervisorType hypervisorType = _volsDao.getHypervisorType(volumeId); + if (hypervisorType.equals(HypervisorType.VMware)) { + try { + boolean isStoragePoolStoragepolicyComplaince = storageMgr.isStoragePoolComplaintWithStoragePolicy(Arrays.asList(vol), destPool); + if (!isStoragePoolStoragepolicyComplaince) { + throw new CloudRuntimeException(String.format("Storage pool %s is not storage policy compliance with the volume %s", poolUuid, vol.getUuid())); + } + } catch (StorageUnavailableException e) { + throw new CloudRuntimeException(String.format("Could not verify storage policy compliance against storage pool %s due to exception %s", destPool.getUuid(), e.getMessage())); + } + } + DiskOfferingVO newDiskOffering = retrieveAndValidateNewDiskOffering(cmd); validateConditionsToReplaceDiskOfferingOfVolume(vol, newDiskOffering, destPool); if (vm != null) { diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java index 9565ac6ff0b..5f61b31171c 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java +++ b/vmware-base/src/main/java/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java @@ -1148,13 +1148,13 @@ public class VirtualMachineMO extends BaseMO { } // vmdkDatastorePath: [datastore name] vmdkFilePath - public void createDisk(String vmdkDatastorePath, long sizeInMb, ManagedObjectReference morDs, int controllerKey) throws Exception { - createDisk(vmdkDatastorePath, VirtualDiskType.THIN, VirtualDiskMode.PERSISTENT, null, sizeInMb, morDs, controllerKey); + public void createDisk(String vmdkDatastorePath, long sizeInMb, ManagedObjectReference morDs, int controllerKey, String vSphereStoragePolicyId) throws Exception { + createDisk(vmdkDatastorePath, VirtualDiskType.THIN, VirtualDiskMode.PERSISTENT, null, sizeInMb, morDs, controllerKey, vSphereStoragePolicyId); } // vmdkDatastorePath: [datastore name] vmdkFilePath public void createDisk(String vmdkDatastorePath, VirtualDiskType diskType, VirtualDiskMode diskMode, String rdmDeviceName, long sizeInMb, - ManagedObjectReference morDs, int controllerKey) throws Exception { + ManagedObjectReference morDs, int controllerKey, String vSphereStoragePolicyId) throws Exception { if (s_logger.isTraceEnabled()) s_logger.trace("vCenter API trace - createDisk(). target MOR: " + _mor.getValue() + ", vmdkDatastorePath: " + vmdkDatastorePath + ", sizeInMb: " + sizeInMb + @@ -1219,7 +1219,14 @@ public class VirtualMachineMO extends BaseMO { deviceConfigSpec.setDevice(newDisk); deviceConfigSpec.setFileOperation(VirtualDeviceConfigSpecFileOperation.CREATE); deviceConfigSpec.setOperation(VirtualDeviceConfigSpecOperation.ADD); - + if (!StringUtils.isEmpty(vSphereStoragePolicyId)) { + PbmProfileManagerMO profMgrMo = new PbmProfileManagerMO(getContext()); + VirtualMachineDefinedProfileSpec diskProfileSpec = profMgrMo.getProfileSpec(vSphereStoragePolicyId); + deviceConfigSpec.getProfile().add(diskProfileSpec); + if (s_logger.isDebugEnabled()) { + s_logger.debug(String.format("Adding vSphere storage profile: %s to volume [%s]", vSphereStoragePolicyId, vmdkDatastorePath)); + } + } reConfigSpec.getDeviceChange().add(deviceConfigSpec); ManagedObjectReference morTask = _context.getService().reconfigVMTask(_mor, reConfigSpec); @@ -1297,7 +1304,7 @@ public class VirtualMachineMO extends BaseMO { attachDisk(vmdkDatastorePathChain, morDs, null, null); } - public void attachDisk(String[] vmdkDatastorePathChain, ManagedObjectReference morDs, String diskController, String storagePolicyId) throws Exception { + public void attachDisk(String[] vmdkDatastorePathChain, ManagedObjectReference morDs, String diskController, String vSphereStoragePolicyId) throws Exception { if(s_logger.isTraceEnabled()) s_logger.trace("vCenter API trace - attachDisk(). target MOR: " + _mor.getValue() + ", vmdkDatastorePath: " @@ -1338,12 +1345,12 @@ public class VirtualMachineMO extends BaseMO { deviceConfigSpec.setDevice(newDisk); deviceConfigSpec.setOperation(VirtualDeviceConfigSpecOperation.ADD); - if (!StringUtils.isEmpty(storagePolicyId)) { + if (!StringUtils.isEmpty(vSphereStoragePolicyId)) { PbmProfileManagerMO profMgrMo = new PbmProfileManagerMO(getContext()); - VirtualMachineDefinedProfileSpec diskProfileSpec = profMgrMo.getProfileSpec(storagePolicyId); + VirtualMachineDefinedProfileSpec diskProfileSpec = profMgrMo.getProfileSpec(vSphereStoragePolicyId); deviceConfigSpec.getProfile().add(diskProfileSpec); if (s_logger.isDebugEnabled()) { - s_logger.debug(String.format("Adding vSphere storage profile: %s to volume [%s]", storagePolicyId, vmdkDatastorePathChain[0])); + s_logger.debug(String.format("Adding vSphere storage profile: %s to volume [%s]", vSphereStoragePolicyId, vmdkDatastorePathChain[0])); } } reConfigSpec.getDeviceChange().add(deviceConfigSpec); From 048e8c8744bf82daf852dbdac42f1bb8bd64e58d Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Mon, 12 Oct 2020 12:27:39 +0530 Subject: [PATCH 133/164] Fixed the issue of VM deletion not cleaning the VM folder on vVols datastore. Fixed it by deleting the VM as complete entity including the extra root disks. --- .../com/cloud/storage/resource/VmwareStorageProcessor.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java index e7349ee49ba..7127031ae9b 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -2670,13 +2670,12 @@ public class VmwareStorageProcessor implements StorageProcessor { List virtualDisks = vmMo.getVirtualDisks(); List managedDatastoreNames = getManagedDatastoreNamesFromVirtualDisks(virtualDisks); - List detachedDisks = vmMo.detachAllDisksExcept(vol.getPath(), diskInfo != null ? diskInfo.getDiskDeviceBusName() : null); - VmwareStorageLayoutHelper.moveVolumeToRootFolder(new DatacenterMO(context, morDc), detachedDisks); - // let vmMo.destroy to delete volume for us // vmMo.tearDownDevices(new Class[] { VirtualDisk.class, VirtualEthernetCard.class }); if (isManaged) { + List detachedDisks = vmMo.detachAllDisksExcept(vol.getPath(), diskInfo != null ? diskInfo.getDiskDeviceBusName() : null); + VmwareStorageLayoutHelper.moveVolumeToRootFolder(new DatacenterMO(context, morDc), detachedDisks); vmMo.unregisterVm(); } else { From 88c02efd2720047fee32788de64d1259e29767f0 Mon Sep 17 00:00:00 2001 From: nvazquez Date: Tue, 13 Oct 2020 12:33:37 -0300 Subject: [PATCH 134/164] Fix deploy-as-is templates GC --- .../cloud/hypervisor/vmware/resource/VmwareResource.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 3e515d7b5cd..275e61bb7e3 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -7165,11 +7165,13 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa VirtualMachineMO vmMo = findVmOnDatacenter(context, hyperHost, vol); - if (vmMo != null && vmMo.isTemplate()) { + if (vmMo != null) { if (s_logger.isInfoEnabled()) { s_logger.info("Destroy template volume " + vol.getPath()); } - vmMo.markAsVirtualMachine(hyperHost.getHyperHostOwnerResourcePool(), hyperHost.getMor()); + if (vmMo.isTemplate()) { + vmMo.markAsVirtualMachine(hyperHost.getHyperHostOwnerResourcePool(), hyperHost.getMor()); + } vmMo.destroy(); } else { if (s_logger.isInfoEnabled()) { From 3600b3c7bc5a03b1abf8ae6dde7c06b6faf2ec04 Mon Sep 17 00:00:00 2001 From: nvazquez Date: Tue, 13 Oct 2020 16:20:38 -0300 Subject: [PATCH 135/164] Reconcile root disk controller fix --- .../orchestration/VolumeOrchestrator.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java index 6254a999031..05732e58411 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java @@ -860,9 +860,28 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati template, vm, owner, deviceId, configurationId); profiles.add(diskProfile); } + + handleRootDiskControllerTpeForDeployAsIs(templateAsIsDisks, vm); return profiles; } + private void handleRootDiskControllerTpeForDeployAsIs(List disksAsIs, VirtualMachine vm) { + if (CollectionUtils.isNotEmpty(disksAsIs)) { + String diskControllerSubType = disksAsIs.get(0).getDiskControllerSubType(); + if (StringUtils.isNotBlank(diskControllerSubType)) { + long vmId = vm.getId(); + UserVmDetailVO detail = userVmDetailsDao.findDetail(vmId, VmDetailConstants.ROOT_DISK_CONTROLLER); + if (detail != null) { + detail.setValue(diskControllerSubType); + userVmDetailsDao.update(detail.getId(), detail); + } else { + detail = new UserVmDetailVO(vmId, VmDetailConstants.ROOT_DISK_CONTROLLER, diskControllerSubType, false); + userVmDetailsDao.persist(detail); + } + } + } + } + private ImageFormat getSupportedImageFormatForCluster(HypervisorType hyperType) { if (hyperType == HypervisorType.XenServer) { return ImageFormat.VHD; From d81f05091837e161972dd687ac5b91387d2b0878 Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Thu, 15 Oct 2020 00:52:14 +0530 Subject: [PATCH 136/164] Check storage policy of disk offering if it suitable for storage pool while using migratevirtualmachinewithvolume API --- .../main/java/com/cloud/vm/UserVmManagerImpl.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index e1b7d2cb7c2..54b990c2ef6 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -261,6 +261,7 @@ import com.cloud.storage.Storage; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.Storage.StoragePoolType; import com.cloud.storage.Storage.TemplateType; +import com.cloud.storage.StorageManager; import com.cloud.storage.StoragePool; import com.cloud.storage.StoragePoolStatus; import com.cloud.storage.VMTemplateStorageResourceAssoc; @@ -510,6 +511,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir private TemplateDeployAsIsDetailsDao templateDeployAsIsDetailsDao; @Inject private UserVmDeployAsIsDetailsDao userVmDeployAsIsDetailsDao; + @Inject + private StorageManager storageMgr; private ScheduledExecutorService _executor = null; private ScheduledExecutorService _vmIpFetchExecutor = null; @@ -6154,6 +6157,17 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir } volToPoolObjectMap.put(Long.valueOf(volume.getId()), Long.valueOf(pool.getId())); } + HypervisorType hypervisorType = _volsDao.getHypervisorType(volume.getId()); + if (hypervisorType.equals(HypervisorType.VMware)) { + try { + boolean isStoragePoolStoragepolicyComplaince = storageMgr.isStoragePoolComplaintWithStoragePolicy(Arrays.asList(volume), pool); + if (!isStoragePoolStoragepolicyComplaince) { + throw new CloudRuntimeException(String.format("Storage pool %s is not storage policy compliance with the volume %s", pool.getUuid(), volume.getUuid())); + } + } catch (StorageUnavailableException e) { + throw new CloudRuntimeException(String.format("Could not verify storage policy compliance against storage pool %s due to exception %s", pool.getUuid(), e.getMessage())); + } + } } } From 950292dcb059f9c76ec196ffb5ce035ec5485277 Mon Sep 17 00:00:00 2001 From: nvazquez Date: Thu, 15 Oct 2020 13:26:34 -0300 Subject: [PATCH 137/164] Ensure deploy as is disks get allocated to the same storage pool --- .../deploy/DeploymentPlanningManagerImpl.java | 158 +++++++++++++----- 1 file changed, 113 insertions(+), 45 deletions(-) diff --git a/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java b/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java index eb99a8eefe3..6c00709e173 100644 --- a/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java +++ b/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java @@ -30,6 +30,8 @@ import java.util.TreeSet; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.dao.VMTemplateDao; import com.cloud.utils.StringUtils; import com.cloud.exception.StorageUnavailableException; import com.cloud.utils.db.Filter; @@ -173,6 +175,8 @@ StateListener { protected VMReservationDao _reservationDao; @Inject HostDetailsDao _hostDetailsDao; + @Inject + private VMTemplateDao templateDao; private static final long INITIAL_RESERVATION_RELEASE_CHECKER_DELAY = 30L * 1000L; // thirty seconds expressed in milliseconds protected long _nodeId = -1; @@ -342,7 +346,7 @@ StateListener { suitableHosts.add(host); Pair> potentialResources = findPotentialDeploymentResources( suitableHosts, suitableVolumeStoragePools, avoids, - getPlannerUsage(planner, vmProfile, plan, avoids), readyAndReusedVolumes, plan.getPreferredHosts()); + getPlannerUsage(planner, vmProfile, plan, avoids), readyAndReusedVolumes, plan.getPreferredHosts(), vm); if (potentialResources != null) { pod = _podDao.findById(host.getPodId()); cluster = _clusterDao.findById(host.getClusterId()); @@ -473,7 +477,7 @@ StateListener { suitableHosts.add(host); Pair> potentialResources = findPotentialDeploymentResources( suitableHosts, suitableVolumeStoragePools, avoids, - getPlannerUsage(planner, vmProfile, plan, avoids), readyAndReusedVolumes, plan.getPreferredHosts()); + getPlannerUsage(planner, vmProfile, plan, avoids), readyAndReusedVolumes, plan.getPreferredHosts(), vm); if (potentialResources != null) { Map storageVolMap = potentialResources.second(); // remove the reused vol<->pool from @@ -557,6 +561,12 @@ StateListener { return dest; } + private boolean isDeployAsIs(VirtualMachine vm) { + long templateId = vm.getTemplateId(); + VMTemplateVO template = templateDao.findById(templateId); + return template != null && template.isDeployAsIs(); + } + /** * Display storage in the logs by default if the template is not deploy-as-is. */ @@ -1120,7 +1130,7 @@ StateListener { // choose the potential host and pool for the VM if (!suitableVolumeStoragePools.isEmpty()) { Pair> potentialResources = findPotentialDeploymentResources(suitableHosts, suitableVolumeStoragePools, avoid, - resourceUsageRequired, readyAndReusedVolumes, plan.getPreferredHosts()); + resourceUsageRequired, readyAndReusedVolumes, plan.getPreferredHosts(), vmProfile.getVirtualMachine()); if (potentialResources != null) { Host host = _hostDao.findById(potentialResources.first().getId()); @@ -1261,7 +1271,7 @@ StateListener { } protected Pair> findPotentialDeploymentResources(List suitableHosts, Map> suitableVolumeStoragePools, - ExcludeList avoid, DeploymentPlanner.PlannerResourceUsage resourceUsageRequired, List readyAndReusedVolumes, List preferredHosts) { + ExcludeList avoid, PlannerResourceUsage resourceUsageRequired, List readyAndReusedVolumes, List preferredHosts, VirtualMachine vm) { s_logger.debug("Trying to find a potenial host and associated storage pools from the suitable host/pool lists for this VM"); boolean hostCanAccessPool = false; @@ -1283,59 +1293,117 @@ StateListener { }); volumesOrderBySizeDesc.addAll(suitableVolumeStoragePools.keySet()); boolean multipleVolume = volumesOrderBySizeDesc.size() > 1; + boolean deployAsIs = isDeployAsIs(vm); for (Host potentialHost : suitableHosts) { Map> volumeAllocationMap = new HashMap>(); - for (Volume vol : volumesOrderBySizeDesc) { - haveEnoughSpace = false; - s_logger.debug("Checking if host: " + potentialHost.getId() + " can access any suitable storage pool for volume: " + vol.getVolumeType()); - List volumePoolList = suitableVolumeStoragePools.get(vol); - hostCanAccessPool = false; - hostAffinityCheck = checkAffinity(potentialHost, preferredHosts); - for (StoragePool potentialSPool : volumePoolList) { - if (hostCanAccessSPool(potentialHost, potentialSPool)) { - hostCanAccessPool = true; - if (multipleVolume && !readyAndReusedVolumes.contains(vol)) { - List requestVolumes = null; - if (volumeAllocationMap.containsKey(potentialSPool)) - requestVolumes = volumeAllocationMap.get(potentialSPool); - else - requestVolumes = new ArrayList(); - requestVolumes.add(vol); + if (deployAsIs) { + storage = new HashMap<>(); + // Find the common suitable pools + s_logger.debug("Trying to allocate all the VM volumes to a single storage pool"); + Set suitablePools = new HashSet<>(); + List notAllowedPools = new ArrayList<>(); + for (List pools : suitableVolumeStoragePools.values()) { + if (CollectionUtils.isEmpty(suitablePools)) { + // All the suitable pools of the first volume + suitablePools.addAll(pools); + } else { + for (StoragePool pool : pools) { + if (!suitablePools.contains(pool)) { + s_logger.debug("Storage pool " + pool.getUuid() + " not allowed for this VM"); + notAllowedPools.add(pool); + } + } + } + } + suitablePools.removeAll(notAllowedPools); + if (CollectionUtils.isEmpty(suitablePools)) { + s_logger.debug("Could not find a storage pool to fit all the VM volumes on this host"); + continue; + } - if (potentialHost.getHypervisorType() == HypervisorType.VMware) { - try { - boolean isStoragePoolStoragepolicyComplaince = _storageMgr.isStoragePoolComplaintWithStoragePolicy(requestVolumes, potentialSPool); - if (!isStoragePoolStoragepolicyComplaince) { - continue; - } - } catch (StorageUnavailableException e) { - s_logger.warn(String.format("Could not verify storage policy complaince against storage pool %s due to exception %s", potentialSPool.getUuid(), e.getMessage())); + List allVolumes = new ArrayList<>(); + allVolumes.addAll(volumesOrderBySizeDesc); + for (StoragePool storagePool : suitablePools) { + haveEnoughSpace = false; + hostCanAccessPool = false; + hostAffinityCheck = checkAffinity(potentialHost, preferredHosts); + if (hostCanAccessSPool(potentialHost, storagePool)) { + hostCanAccessPool = true; + if (potentialHost.getHypervisorType() == HypervisorType.VMware) { + try { + boolean isStoragePoolStoragepolicyComplaince = _storageMgr.isStoragePoolComplaintWithStoragePolicy(allVolumes, storagePool); + if (!isStoragePoolStoragepolicyComplaince) { continue; } - } - - if (!_storageMgr.storagePoolHasEnoughIops(requestVolumes, potentialSPool) || - !_storageMgr.storagePoolHasEnoughSpace(requestVolumes, potentialSPool, potentialHost.getClusterId())) + } catch (StorageUnavailableException e) { + s_logger.warn(String.format("Could not verify storage policy complaince against storage pool %s due to exception %s", storagePool.getUuid(), e.getMessage())); continue; - volumeAllocationMap.put(potentialSPool, requestVolumes); + } + haveEnoughSpace = true; + } + } + if (hostCanAccessPool && haveEnoughSpace && hostAffinityCheck) { + for (Volume vol : volumesOrderBySizeDesc) { + s_logger.debug("Found a suitable storage pool for all the VM volumes: " + storagePool.getUuid()); + storage.put(vol, storagePool); } - storage.put(vol, potentialSPool); - haveEnoughSpace = true; break; } } - if (!hostCanAccessPool) { - break; - } - if (!haveEnoughSpace) { - s_logger.warn("insufficient capacity to allocate all volumes"); - break; - } - if (!hostAffinityCheck) { - s_logger.debug("Host affinity check failed"); - break; + } else { + for (Volume vol : volumesOrderBySizeDesc) { + haveEnoughSpace = false; + s_logger.debug("Checking if host: " + potentialHost.getId() + " can access any suitable storage pool for volume: " + vol.getVolumeType()); + List volumePoolList = suitableVolumeStoragePools.get(vol); + hostCanAccessPool = false; + hostAffinityCheck = checkAffinity(potentialHost, preferredHosts); + for (StoragePool potentialSPool : volumePoolList) { + if (hostCanAccessSPool(potentialHost, potentialSPool)) { + hostCanAccessPool = true; + if (multipleVolume && !readyAndReusedVolumes.contains(vol)) { + List requestVolumes = null; + if (volumeAllocationMap.containsKey(potentialSPool)) + requestVolumes = volumeAllocationMap.get(potentialSPool); + else + requestVolumes = new ArrayList(); + requestVolumes.add(vol); + + if (potentialHost.getHypervisorType() == HypervisorType.VMware) { + try { + boolean isStoragePoolStoragepolicyComplaince = _storageMgr.isStoragePoolComplaintWithStoragePolicy(requestVolumes, potentialSPool); + if (!isStoragePoolStoragepolicyComplaince) { + continue; + } + } catch (StorageUnavailableException e) { + s_logger.warn(String.format("Could not verify storage policy complaince against storage pool %s due to exception %s", potentialSPool.getUuid(), e.getMessage())); + continue; + } + } + + if (!_storageMgr.storagePoolHasEnoughIops(requestVolumes, potentialSPool) || + !_storageMgr.storagePoolHasEnoughSpace(requestVolumes, potentialSPool, potentialHost.getClusterId())) + continue; + volumeAllocationMap.put(potentialSPool, requestVolumes); + } + storage.put(vol, potentialSPool); + haveEnoughSpace = true; + break; + } + } + if (!hostCanAccessPool) { + break; + } + if (!haveEnoughSpace) { + s_logger.warn("insufficient capacity to allocate all volumes"); + break; + } + if (!hostAffinityCheck) { + s_logger.debug("Host affinity check failed"); + break; + } } } + if (hostCanAccessPool && haveEnoughSpace && hostAffinityCheck && checkIfHostFitsPlannerUsage(potentialHost.getId(), resourceUsageRequired)) { s_logger.debug("Found a potential host " + "id: " + potentialHost.getId() + " name: " + potentialHost.getName() + " and associated storage pools for this VM"); From ee5b8763a6b7cafee7310304dcaad30bfbd8f453 Mon Sep 17 00:00:00 2001 From: nvazquez Date: Fri, 16 Oct 2020 00:47:34 -0300 Subject: [PATCH 138/164] Fix remove VM and its volumes for deploy-as-is if have previously failed - restore cpu flags in nested virtualization test --- .../storage/volume/VolumeServiceImpl.java | 13 ++++++++++--- .../integration/smoke/test_nested_virtualization.py | 2 +- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java index 53a36ede6bd..5e3493a29f1 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java @@ -28,6 +28,8 @@ import java.util.Random; import javax.inject.Inject; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.dao.VMTemplateDao; import org.apache.cloudstack.engine.cloud.entity.api.VolumeEntity; import org.apache.cloudstack.engine.subsystem.api.storage.ChapInfo; import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; @@ -168,6 +170,8 @@ public class VolumeServiceImpl implements VolumeService { private ClusterDao clusterDao; @Inject private VolumeDetailsDao _volumeDetailsDao; + @Inject + private VMTemplateDao templateDao; private final static String SNAPSHOT_ID = "SNAPSHOT_ID"; @@ -351,9 +355,12 @@ public class VolumeServiceImpl implements VolumeService { if (s_logger.isDebugEnabled()) { s_logger.debug("Marking volume that was never created as destroyed: " + vol); } - volDao.remove(vol.getId()); - future.complete(result); - return future; + VMTemplateVO template = templateDao.findById(vol.getTemplateId()); + if (template != null && !template.isDeployAsIs()) { + volDao.remove(vol.getId()); + future.complete(result); + return future; + } } } VolumeObject vo = (VolumeObject)volume; diff --git a/test/integration/smoke/test_nested_virtualization.py b/test/integration/smoke/test_nested_virtualization.py index 43313244f41..10dcf895ed9 100644 --- a/test/integration/smoke/test_nested_virtualization.py +++ b/test/integration/smoke/test_nested_virtualization.py @@ -139,7 +139,7 @@ class TestNestedVirtualization(cloudstackTestCase): #5) Check for CPU flags: vmx for Intel and svm for AMD indicates nested virtualization is enabled self.assert_(result is not None, "Empty result for CPU flags") res = str(result) - self.assertTrue('vmx' in res or 'svm' in res or 'lm' in res) + self.assertTrue('vmx' in res or 'svm' in res) except Exception as e: self.debug('Error=%s' % e) self.rollback_nested_configurations(rollback_nv, rollback_nv_per_vm) From 38e119a09998a62fb40ae78fc56bec34f8af77bc Mon Sep 17 00:00:00 2001 From: davidjumani Date: Fri, 16 Oct 2020 18:05:08 +0530 Subject: [PATCH 139/164] Throwing error if ostypeid, nicAdapter, rootDiskController is passed for VMWare templates --- .../user/template/RegisterTemplateCmd.java | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java index 900c02e7c01..a26dece4d21 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java @@ -43,6 +43,7 @@ import org.apache.log4j.Logger; import com.cloud.exception.ResourceAllocationException; import com.cloud.template.VirtualMachineTemplate; +import com.cloud.vm.VmDetailConstants; @APICommand(name = "registerTemplate", description = "Registers an existing template into the CloudStack cloud. ", responseObject = TemplateResponse.class, responseView = ResponseView.Restricted, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) @@ -87,7 +88,7 @@ public class RegisterTemplateCmd extends BaseCmd implements UserCmd { type = CommandType.UUID, entityType = GuestOSResponse.class, required = true, - description = "the ID of the OS Type that best represents the OS of this template.") + description = "the ID of the OS Type that best represents the OS of this template. Not applicable with VMware, as we honour what is defined in the template") private Long osTypeId; @Parameter(name = ApiConstants.PASSWORD_ENABLED, @@ -340,5 +341,19 @@ public class RegisterTemplateCmd extends BaseCmd implements UserCmd { throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Parameter directdownload is only allowed for KVM templates"); } + + if (getHypervisor().equalsIgnoreCase(Hypervisor.HypervisorType.VMware.toString())) { + if (osTypeId != null) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "ostypeid is not supported on VMWare"); + } + + details = getDetails(); + if (details.containsKey(VmDetailConstants.ROOT_DISK_CONTROLLER)) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "rootDiskController is not supported on VMWare"); + } + if (details.containsKey(VmDetailConstants.NIC_ADAPTER)) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "nicAdapter is not supported on VMWare"); + } + } } } From 6b715ecb532f22a67337d803408e48df68c0dc23 Mon Sep 17 00:00:00 2001 From: davidjumani Date: Fri, 16 Oct 2020 18:09:38 +0530 Subject: [PATCH 140/164] Throwing error if boot type or boot mode specified while deploying VMs on VMware --- .../apache/cloudstack/api/command/user/vm/DeployVMCmd.java | 7 +++---- server/src/main/java/com/cloud/vm/UserVmManagerImpl.java | 4 ++++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java index 09455d45dff..6d7cc9c81c2 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java @@ -116,10 +116,10 @@ public class DeployVMCmd extends BaseAsyncCreateCustomIdCmd implements SecurityG @Parameter(name = ApiConstants.NETWORK_IDS, type = CommandType.LIST, collectionType = CommandType.UUID, entityType = NetworkResponse.class, description = "list of network ids used by virtual machine. Can't be specified with ipToNetworkList parameter") private List networkIds; - @Parameter(name = ApiConstants.BOOT_TYPE, type = CommandType.STRING, required = false, description = "Guest VM Boot option either custom[UEFI] or default boot [BIOS]", since = "4.14.0.0") + @Parameter(name = ApiConstants.BOOT_TYPE, type = CommandType.STRING, required = false, description = "Guest VM Boot option either custom[UEFI] or default boot [BIOS]. Not applicable with VMware, as we honour what is defined in the template.", since = "4.14.0.0") private String bootType; - @Parameter(name = ApiConstants.BOOT_MODE, type = CommandType.STRING, required = false, description = "Boot Mode [Legacy] or [Secure] Applicable when Boot Type Selected is UEFI, otherwise Legacy By default for BIOS", since = "4.14.0.0") + @Parameter(name = ApiConstants.BOOT_MODE, type = CommandType.STRING, required = false, description = "Boot Mode [Legacy] or [Secure] Applicable when Boot Type Selected is UEFI, otherwise Legacy only for BIOS. Not applicable with VMware, as we honour what is defined in the template.", since = "4.14.0.0") private String bootMode; @Parameter(name = ApiConstants.BOOT_INTO_SETUP, type = CommandType.BOOLEAN, required = false, description = "Boot into hardware setup or not (ignored if startVm = false, only valid for vmware)", since = "4.15.0.0") @@ -265,8 +265,7 @@ public class DeployVMCmd extends BaseAsyncCreateCustomIdCmd implements SecurityG return domainId; } - private ApiConstants.BootType getBootType() { - + public ApiConstants.BootType getBootType() { if (StringUtils.isNotBlank(bootType)) { try { String type = bootType.trim().toUpperCase(); diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 54b990c2ef6..4cd7043768d 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -5184,6 +5184,10 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir if (template == null) { throw new InvalidParameterValueException("Unable to use template " + templateId); } + // Bootmode and boottype are not supported on VMWare + if (template.getHypervisorType() == HypervisorType.VMware && (cmd.getBootMode() != null || cmd.getBootType() != null)) { + throw new InvalidParameterValueException("Boot type and boot mode are not supported on VMware, as we honour what is defined in the template."); + } Long diskOfferingId = cmd.getDiskOfferingId(); DiskOffering diskOffering = null; From b8a79aba9222f3b9bce865fb8fd4fca475b1bb63 Mon Sep 17 00:00:00 2001 From: davidjumani Date: Fri, 16 Oct 2020 18:16:28 +0530 Subject: [PATCH 141/164] Elaborating error --- .../api/command/user/template/RegisterTemplateCmd.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java index a26dece4d21..67894ff9f87 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java @@ -344,15 +344,15 @@ public class RegisterTemplateCmd extends BaseCmd implements UserCmd { if (getHypervisor().equalsIgnoreCase(Hypervisor.HypervisorType.VMware.toString())) { if (osTypeId != null) { - throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "ostypeid is not supported on VMWare"); + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "ostypeid is not supported on VMWare, as we honour what is defined in the template"); } details = getDetails(); if (details.containsKey(VmDetailConstants.ROOT_DISK_CONTROLLER)) { - throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "rootDiskController is not supported on VMWare"); + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "rootDiskController is not supported on VMWare, as we honour what is defined in the template"); } if (details.containsKey(VmDetailConstants.NIC_ADAPTER)) { - throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "nicAdapter is not supported on VMWare"); + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "nicAdapter is not supported on VMWare, as we honour what is defined in the template"); } } } From 6e6d6d60b7946f5c4217b46aaf22442449936d5c Mon Sep 17 00:00:00 2001 From: nvazquez Date: Sun, 18 Oct 2020 23:56:42 -0300 Subject: [PATCH 142/164] Fix template registration --- .../user/template/RegisterTemplateCmd.java | 19 ++++++++++++------- .../java/com/cloud/vm/UserVmManagerImpl.java | 5 +++-- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java index 67894ff9f87..bcb274daded 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java @@ -39,6 +39,7 @@ import org.apache.cloudstack.api.response.ProjectResponse; import org.apache.cloudstack.api.response.TemplateResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.cloudstack.context.CallContext; +import org.apache.commons.collections.MapUtils; import org.apache.log4j.Logger; import com.cloud.exception.ResourceAllocationException; @@ -87,7 +88,7 @@ public class RegisterTemplateCmd extends BaseCmd implements UserCmd { @Parameter(name = ApiConstants.OS_TYPE_ID, type = CommandType.UUID, entityType = GuestOSResponse.class, - required = true, + required = false, description = "the ID of the OS Type that best represents the OS of this template. Not applicable with VMware, as we honour what is defined in the template") private Long osTypeId; @@ -347,13 +348,17 @@ public class RegisterTemplateCmd extends BaseCmd implements UserCmd { throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "ostypeid is not supported on VMWare, as we honour what is defined in the template"); } - details = getDetails(); - if (details.containsKey(VmDetailConstants.ROOT_DISK_CONTROLLER)) { - throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "rootDiskController is not supported on VMWare, as we honour what is defined in the template"); - } - if (details.containsKey(VmDetailConstants.NIC_ADAPTER)) { - throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "nicAdapter is not supported on VMWare, as we honour what is defined in the template"); + Map templateDetails = getDetails(); + if (MapUtils.isNotEmpty(templateDetails)) { + if (templateDetails.containsKey(VmDetailConstants.ROOT_DISK_CONTROLLER)) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "rootDiskController is not supported on VMWare, as we honour what is defined in the template"); + } + if (templateDetails.containsKey(VmDetailConstants.NIC_ADAPTER)) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "nicAdapter is not supported on VMWare, as we honour what is defined in the template"); + } } + } else if (osTypeId == null) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Please provide a guest OS type"); } } } diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 4cd7043768d..227ee5abbbc 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -5184,8 +5184,9 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir if (template == null) { throw new InvalidParameterValueException("Unable to use template " + templateId); } - // Bootmode and boottype are not supported on VMWare - if (template.getHypervisorType() == HypervisorType.VMware && (cmd.getBootMode() != null || cmd.getBootType() != null)) { + + // Bootmode and boottype are not supported on VMWare dpeloy-as-is templates (since 4.15) + if (template.isDeployAsIs() && (cmd.getBootMode() != null || cmd.getBootType() != null)) { throw new InvalidParameterValueException("Boot type and boot mode are not supported on VMware, as we honour what is defined in the template."); } From 50ab1b2fbe84aa60d8d7adc12919bebc83b5e003 Mon Sep 17 00:00:00 2001 From: nvazquez Date: Mon, 19 Oct 2020 00:09:22 -0300 Subject: [PATCH 143/164] Add missing guest OS entries for VMware --- .../META-INF/db/schema-41400to41500.sql | 268 +++++++++++++++++- 1 file changed, 267 insertions(+), 1 deletion(-) diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql b/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql index 4df87b8cb33..055a108f60d 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql @@ -193,7 +193,7 @@ UPDATE `cloud`.`guest_os_hypervisor` SET guest_os_name='CentOS 7' where guest_os INSERT IGNORE INTO `cloud`.`hypervisor_capabilities`(uuid, hypervisor_type, hypervisor_version, max_guests_limit, max_data_volumes_limit, max_hosts_per_cluster, storage_motion_supported) values (UUID(), 'XenServer', '8.1.0', 1000, 253, 64, 1); -- Copy XenServer 8.0 hypervisor guest OS mappings to XenServer8.1 -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) SELECT UUID(),'Xenserver', '8.1.0', guest_os_name, guest_os_id, utc_timestamp(), 0 FROM `cloud`.`guest_os_hypervisor` WHERE hypervisor_type='Xenserver' AND hypervisor_version='8.0.0'; +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) SELECT UUID(),'Xenserver', '8.1.0', guest_os_name, guest_os_id, now(), 0 FROM `cloud`.`guest_os_hypervisor` WHERE hypervisor_type='Xenserver' AND hypervisor_version='8.0.0'; CREATE TABLE IF NOT EXISTS `cloud`.`vsphere_storage_policy` ( `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT, @@ -600,3 +600,269 @@ INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid, hypervisor_type, hypervi INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid, hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(), 'VMware', '6.7.1', 'other3xLinuxGuest', 307, now(), 0); INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid, hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(), 'VMware', '6.7.2', 'other3xLinuxGuest', 307, now(), 0); INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid, hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(), 'VMware', '6.7.3', 'other3xLinuxGuest', 307, now(), 0); + + +-- Add amazonlinux as support guest os +INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (308, UUID(), 7, 'Amazon Linux 2 (64 bit)', now()); +-- Amazonlinux VMWare guest os mapping +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'amazonlinux2_64Guest', 308, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'amazonlinux2_64Guest', 308, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'amazonlinux2_64Guest', 308, now(), 0); + + +-- Add asianux4 32 as support guest os +INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (309, UUID(), 7, 'Asianux Server 4 (32 bit)', now()); +-- asianux4 VMWare guest os mapping +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '4.0', 'asianux4Guest', 309, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '4.1', 'asianux4Guest', 309, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.0', 'asianux4Guest', 309, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.1', 'asianux4Guest', 309, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.5', 'asianux4Guest', 309, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.0', 'asianux4Guest', 309, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'asianux4Guest', 309, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'asianux4Guest', 309, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'asianux4Guest', 309, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'asianux4Guest', 309, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'asianux4Guest', 309, now(), 0); + + +-- Add asianux4 64 as support guest os +INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (310, UUID(), 7, 'Asianux Server 4 (64 bit)', now()); +-- asianux4 VMWare guest os mapping +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '4.0', 'asianux4_64Guest', 310, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '4.1', 'asianux4_64Guest', 310, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.0', 'asianux4_64Guest', 310, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.1', 'asianux4_64Guest', 310, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.5', 'asianux4_64Guest', 310, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.0', 'asianux4_64Guest', 310, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'asianux4_64Guest', 310, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'asianux4_64Guest', 310, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'asianux4_64Guest', 310, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'asianux4_64Guest', 310, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'asianux4_64Guest', 310, now(), 0); + + +-- Add asianux5 32 as support guest os +INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (311, UUID(), 7, 'Asianux Server 5 (32 bit)', now()); +-- asianux5 VMWare guest os mapping +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.0', 'asianux5Guest', 311, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'asianux5Guest', 311, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'asianux5Guest', 311, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'asianux5Guest', 311, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'asianux5Guest', 311, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'asianux5Guest', 311, now(), 0); + + +-- Add asianux5 64 as support guest os +INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (312, UUID(), 7, 'Asianux Server 5 (64 bit)', now()); +-- asianux5 VMWare guest os mapping +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.0', 'asianux5_64Guest', 312, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'asianux5_64Guest', 312, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'asianux5_64Guest', 312, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'asianux5_64Guest', 312, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'asianux5_64Guest', 312, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'asianux5_64Guest', 312, now(), 0); + + +-- Add asianux7 32 as support guest os +INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (313, UUID(), 7, 'Asianux Server 7 (32 bit)', now()); +-- asianux7 VMWare guest os mapping +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'asianux7Guest', 313, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'asianux7Guest', 313, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'asianux7Guest', 313, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'asianux7Guest', 313, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'asianux7Guest', 313, now(), 0); + + +-- Add asianux7 64 as support guest os +INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (314, UUID(), 7, 'Asianux Server 7 (64 bit)', now()); +-- asianux7 VMWare guest os mapping +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'asianux7_64Guest', 314, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'asianux7_64Guest', 314, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'asianux7_64Guest', 314, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'asianux7_64Guest', 314, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'asianux7_64Guest', 314, now(), 0); + + +-- Add asianux8 as support guest os +INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (315, UUID(), 7, 'Asianux Server 8 (64 bit)', now()); +-- asianux8 VMWare guest os mapping +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'asianux8_64Guest', 315, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'asianux8_64Guest', 315, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'asianux8_64Guest', 315, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'asianux8_64Guest', 315, now(), 0); + + +-- Add eComStation 2.0 as support guest os +INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (316, UUID(), 7, 'eComStation 2.0', now()); +-- eComStation 2.0 VMWare guest os mapping +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'eComStation2Guest', 316, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'eComStation2Guest', 316, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'eComStation2Guest', 316, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'eComStation2Guest', 316, now(), 0); + +-- Add macOS 10.13 (64 bit) as support guest os +INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (317, UUID(), 7, 'macOS 10.13 (64 bit)', now()); +-- macOS 10.13 (64 bit) VMWare guest os mapping +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'darwin17_64Guest', 317, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'darwin17_64Guest', 317, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'darwin17_64Guest', 317, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'darwin17_64Guest', 317, now(), 0); + +-- Add macOS 10.14 (64 bit) as support guest os +INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (318, UUID(), 7, 'macOS 10.14 (64 bit)', now()); +-- macOS 10.14 (64 bit) VMWare guest os mapping +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'darwin18_64Guest', 318, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'darwin18_64Guest', 318, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'darwin18_64Guest', 318, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'darwin18_64Guest', 318, now(), 0); + + +-- Add Fedora Linux (64 bit) as support guest os +INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (319, UUID(), 7, 'Fedora Linux (64 bit)', now()); +-- Fedora Linux (64 bit) VMWare guest os mapping +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.1', 'fedora64Guest', 319, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.5', 'fedora64Guest', 319, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.0', 'fedora64Guest', 319, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'fedora64Guest', 319, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'fedora64Guest', 319, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'fedora64Guest', 319, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'fedora64Guest', 319, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'fedora64Guest', 319, now(), 0); + + +-- Add Fedora Linux as support guest os +INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (320, UUID(), 7, 'Fedora Linux', now()); +-- Fedora Linux VMWare guest os mapping +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.1', 'fedoraGuest', 320, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.5', 'fedoraGuest', 320, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.0', 'fedoraGuest', 320, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'fedoraGuest', 320, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'fedoraGuest', 320, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'fedoraGuest', 320, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'fedoraGuest', 320, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'fedoraGuest', 320, now(), 0); + +-- Add Mandrake Linux as support guest os +INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (321, UUID(), 7, 'Mandrake Linux', now()); +-- Mandrake Linux VMWare guest os mapping +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.5', 'mandrakeGuest', 321, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.0', 'mandrakeGuest', 321, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'mandrakeGuest', 321, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'mandrakeGuest', 321, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'mandrakeGuest', 321, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'mandrakeGuest', 321, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'mandrakeGuest', 321, now(), 0); + +-- Add Mandriva Linux (64 bit) as support guest os +INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (322, UUID(), 7, 'Mandriva Linux (64 bit)', now()); +-- Mandriva Linux (64 bit) VMWare guest os mapping +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '4.0', 'mandriva64Guest', 322, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '4.1', 'mandriva64Guest', 322, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.0', 'mandriva64Guest', 322, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.1', 'mandriva64Guest', 322, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.5', 'mandriva64Guest', 322, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.0', 'mandriva64Guest', 322, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'mandriva64Guest', 322, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'mandriva64Guest', 322, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'mandriva64Guest', 322, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'mandriva64Guest', 322, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'mandriva64Guest', 322, now(), 0); + + +-- Add Mandriva Linux as support guest os +INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (323, UUID(), 7, 'Mandriva Linux', now()); +-- Mandriva Linux VMWare guest os mapping +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '4.0', 'mandrivaGuest', 323, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '4.1', 'mandrivaGuest', 323, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.0', 'mandrivaGuest', 323, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.1', 'mandrivaGuest', 323, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.5', 'mandrivaGuest', 323, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.0', 'mandrivaGuest', 323, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'mandrivaGuest', 323, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'mandrivaGuest', 323, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'mandrivaGuest', 323, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'mandrivaGuest', 323, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'mandrivaGuest', 323, now(), 0); + + +-- Add SCO OpenServer 5 as support guest os +INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (324, UUID(), 7, 'SCO OpenServer 5', now()); +-- SCO OpenServer 5 VMWare guest os mapping +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '4.0', 'openServer5Guest', 324, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '4.1', 'openServer5Guest', 324, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.0', 'openServer5Guest', 324, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.1', 'openServer5Guest', 324, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.5', 'openServer5Guest', 324, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.0', 'openServer5Guest', 324, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'openServer5Guest', 324, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'openServer5Guest', 324, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'openServer5Guest', 324, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'openServer5Guest', 324, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'openServer5Guest', 324, now(), 0); + + +-- Add SCO OpenServer 6 as support guest os +INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (325, UUID(), 7, 'SCO OpenServer 6', now()); +-- SCO OpenServer 6 VMWare guest os mapping +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '4.0', 'openServer6Guest', 325, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '4.1', 'openServer6Guest', 325, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.0', 'openServer6Guest', 325, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.1', 'openServer6Guest', 325, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.5', 'openServer6Guest', 325, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.0', 'openServer6Guest', 325, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'openServer6Guest', 325, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'openServer6Guest', 325, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'openServer6Guest', 325, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'openServer6Guest', 325, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'openServer6Guest', 325, now(), 0); + + + +-- Add OpenSUSE Linux (64 bit) as support guest os +INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (326, UUID(), 7, 'OpenSUSE Linux (64 bit)', now()); +-- OpenSUSE Linux (64 bit) VMWare guest os mapping +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.1', 'opensuse64Guest', 326, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.5', 'opensuse64Guest', 326, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.0', 'opensuse64Guest', 326, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'opensuse64Guest', 326, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'opensuse64Guest', 326, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'opensuse64Guest', 326, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'opensuse64Guest', 326, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'opensuse64Guest', 326, now(), 0); + + +-- Add SCO OpenServer 6 as support guest os +INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (327, UUID(), 7, 'SCO OpenServer 6', now()); +-- SCO OpenServer 6 VMWare guest os mapping +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.1', 'opensuseGuest', 327, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.5', 'opensuseGuest', 327, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.0', 'opensuseGuest', 327, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'opensuseGuest', 327, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'opensuseGuest', 327, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'opensuseGuest', 327, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'opensuseGuest', 327, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'opensuseGuest', 327, now(), 0); + + +-- Add Solaris 11 (64 bit) as support guest os +INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (328, UUID(), 7, 'Solaris 11 (64 bit)', now()); +-- Solaris 11 (64 bit) VMWare guest os mapping +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.1', 'solaris11_64Guest', 328, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.5', 'solaris11_64Guest', 328, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.0', 'solaris11_64Guest', 328, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'solaris11_64Guest', 328, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'solaris11_64Guest', 328, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'solaris11_64Guest', 328, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'solaris11_64Guest', 328, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'solaris11_64Guest', 328, now(), 0); + + +-- Add VMware Photon (64 bit) as support guest os +INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (329, UUID(), 7, 'VMware Photon (64 bit)', now()); +-- VMware Photon (64 bit) VMWare guest os mapping +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'vmwarePhoton64Guest', 329, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'vmwarePhoton64Guest', 329, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'vmwarePhoton64Guest', 329, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'vmwarePhoton64Guest', 329, now(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'vmwarePhoton64Guest', 329, now(), 0); From 67794aba232c592e685c56b4089e5f7bd893f43c Mon Sep 17 00:00:00 2001 From: nvazquez Date: Tue, 20 Oct 2020 03:00:21 -0300 Subject: [PATCH 144/164] Fix reinstall VM bug --- .../java/com/cloud/agent/api/to/DeployAsIsInfoTO.java | 2 +- .../cloud/hypervisor/vmware/resource/VmwareResource.java | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/api/src/main/java/com/cloud/agent/api/to/DeployAsIsInfoTO.java b/api/src/main/java/com/cloud/agent/api/to/DeployAsIsInfoTO.java index 1c5bf999d84..37d481ea20a 100644 --- a/api/src/main/java/com/cloud/agent/api/to/DeployAsIsInfoTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/DeployAsIsInfoTO.java @@ -31,7 +31,7 @@ public class DeployAsIsInfoTO { @LogLevel(LogLevel.Log4jLevel.Off) private Map properties = new HashMap<>(); private Map nicAdapterMap = new HashMap(); - boolean replaceVm; + private boolean replaceVm; public DeployAsIsInfoTO() { } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 275e61bb7e3..544d779daf2 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -1771,12 +1771,14 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa auxVM.destroy(); String vmNameInVcenter = virtualMachineMO.getName(); virtualMachineMO.tearDownDevices(new Class[]{VirtualDisk.class}); + s_logger.info("Changing VM datastore to " + dsMo); + virtualMachineMO.changeDatastore(morDatastore); for (String vmdkFileBaseName : vmdkFileBaseNames) { - String newPath = null; if (dsMo.folderExists(String.format("[%s]", dsMo.getName()), vmNameInVcenter)) { - newPath = VmwareStorageLayoutHelper.syncVolumeToVmDefaultFolder(dcMo, vmNameInVcenter, dsMo, vmdkFileBaseName, null); + String newPath = VmwareStorageLayoutHelper.syncVolumeToVmDefaultFolder(dcMo, vmNameInVcenter, dsMo, vmdkFileBaseName, null); + s_logger.info("Attaching disk to restored VM at: " + newPath + " on datastore: " + destDatastore); + virtualMachineMO.attachDisk(new String[] {newPath}, morDatastore); } - virtualMachineMO.attachDisk(new String[] {newPath}, morDatastore); } } From 5797cf5e170c9b42e5d265c76126768c28b416dc Mon Sep 17 00:00:00 2001 From: Andrija Panic <45762285+andrijapanicsb@users.noreply.github.com> Date: Tue, 20 Oct 2020 15:40:03 +0200 Subject: [PATCH 145/164] guest-os-GC (#67) --- .../META-INF/db/schema-41400to41500.sql | 355 ++++++++---------- 1 file changed, 157 insertions(+), 198 deletions(-) diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql b/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql index 055a108f60d..bd4d1ca146a 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41400to41500.sql @@ -190,10 +190,10 @@ ALTER TABLE `cloud`.`nics` MODIFY COLUMN update_time timestamp DEFAULT CURRENT_T UPDATE `cloud`.`guest_os_hypervisor` SET guest_os_name='CentOS 7' where guest_os_id=(SELECT guest_os_id from `cloud`.`vm_template` WHERE unique_name='centos56-x86_64-xen') AND hypervisor_type='Xenserver' AND hypervisor_version='8.0.0'; -- Add XenServer 8.1 hypervisor capabilities -INSERT IGNORE INTO `cloud`.`hypervisor_capabilities`(uuid, hypervisor_type, hypervisor_version, max_guests_limit, max_data_volumes_limit, max_hosts_per_cluster, storage_motion_supported) values (UUID(), 'XenServer', '8.1.0', 1000, 253, 64, 1); +INSERT INTO `cloud`.`hypervisor_capabilities`(uuid, hypervisor_type, hypervisor_version, max_guests_limit, max_data_volumes_limit, max_hosts_per_cluster, storage_motion_supported) values (UUID(), 'XenServer', '8.1.0', 1000, 253, 64, 1); -- Copy XenServer 8.0 hypervisor guest OS mappings to XenServer8.1 -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) SELECT UUID(),'Xenserver', '8.1.0', guest_os_name, guest_os_id, now(), 0 FROM `cloud`.`guest_os_hypervisor` WHERE hypervisor_type='Xenserver' AND hypervisor_version='8.0.0'; +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) SELECT UUID(),'Xenserver', '8.1.0', guest_os_name, guest_os_id, now(), 0 FROM `cloud`.`guest_os_hypervisor` WHERE hypervisor_type='Xenserver' AND hypervisor_version='8.0.0'; CREATE TABLE IF NOT EXISTS `cloud`.`vsphere_storage_policy` ( `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT, @@ -583,286 +583,245 @@ ALTER VIEW `cloud`.`image_store_view` AS `cloud`.`image_store_details` ON image_store_details.store_id = image_store.id; -- OVF configured OS while registering deploy-as-is templates Linux 3.x Kernel OS -INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (305, UUID(), 11, 'OVF Configured OS', now()); -INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (306, UUID(), 2, 'Linux 3.x Kernel (64 bit)', now()); -INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (307, UUID(), 2, 'Linux 3.x Kernel (32 bit)', now()); +INSERT INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (305, UUID(), 11, 'OVF Configured OS', now()); +INSERT INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (306, UUID(), 2, 'Linux 3.x Kernel (64 bit)', now()); +INSERT INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (307, UUID(), 2, 'Linux 3.x Kernel (32 bit)', now()); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid, hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(), 'VMware', '6.0', 'other3xLinux64Guest', 306, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid, hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(), 'VMware', '6.5', 'other3xLinux64Guest', 306, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid, hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(), 'VMware', '6.7', 'other3xLinux64Guest', 306, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid, hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(), 'VMware', '6.7.1', 'other3xLinux64Guest', 306, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid, hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(), 'VMware', '6.7.2', 'other3xLinux64Guest', 306, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid, hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(), 'VMware', '6.7.3', 'other3xLinux64Guest', 306, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid, hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(), 'VMware', '6.0', 'other3xLinux64Guest', 306, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid, hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(), 'VMware', '6.5', 'other3xLinux64Guest', 306, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid, hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(), 'VMware', '6.7', 'other3xLinux64Guest', 306, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid, hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(), 'VMware', '6.7.1', 'other3xLinux64Guest', 306, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid, hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(), 'VMware', '6.7.2', 'other3xLinux64Guest', 306, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid, hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(), 'VMware', '6.7.3', 'other3xLinux64Guest', 306, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid, hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(), 'VMware', '6.0', 'other3xLinuxGuest', 307, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid, hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(), 'VMware', '6.5', 'other3xLinuxGuest', 307, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid, hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(), 'VMware', '6.7', 'other3xLinuxGuest', 307, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid, hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(), 'VMware', '6.7.1', 'other3xLinuxGuest', 307, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid, hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(), 'VMware', '6.7.2', 'other3xLinuxGuest', 307, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid, hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(), 'VMware', '6.7.3', 'other3xLinuxGuest', 307, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid, hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(), 'VMware', '6.0', 'other3xLinuxGuest', 307, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid, hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(), 'VMware', '6.5', 'other3xLinuxGuest', 307, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid, hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(), 'VMware', '6.7', 'other3xLinuxGuest', 307, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid, hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(), 'VMware', '6.7.1', 'other3xLinuxGuest', 307, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid, hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(), 'VMware', '6.7.2', 'other3xLinuxGuest', 307, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid, hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(), 'VMware', '6.7.3', 'other3xLinuxGuest', 307, now(), 0); -- Add amazonlinux as support guest os -INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (308, UUID(), 7, 'Amazon Linux 2 (64 bit)', now()); +INSERT INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (308, UUID(), 7, 'Amazon Linux 2 (64 bit)', now()); -- Amazonlinux VMWare guest os mapping -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'amazonlinux2_64Guest', 308, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'amazonlinux2_64Guest', 308, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'amazonlinux2_64Guest', 308, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'amazonlinux2_64Guest', 308, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'amazonlinux2_64Guest', 308, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'amazonlinux2_64Guest', 308, now(), 0); -- Add asianux4 32 as support guest os -INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (309, UUID(), 7, 'Asianux Server 4 (32 bit)', now()); +INSERT INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (309, UUID(), 7, 'Asianux Server 4 (32 bit)', now()); -- asianux4 VMWare guest os mapping -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '4.0', 'asianux4Guest', 309, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '4.1', 'asianux4Guest', 309, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.0', 'asianux4Guest', 309, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.1', 'asianux4Guest', 309, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.5', 'asianux4Guest', 309, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.0', 'asianux4Guest', 309, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'asianux4Guest', 309, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'asianux4Guest', 309, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'asianux4Guest', 309, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'asianux4Guest', 309, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'asianux4Guest', 309, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.0', 'asianux4Guest', 309, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'asianux4Guest', 309, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'asianux4Guest', 309, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'asianux4Guest', 309, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'asianux4Guest', 309, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'asianux4Guest', 309, now(), 0); -- Add asianux4 64 as support guest os -INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (310, UUID(), 7, 'Asianux Server 4 (64 bit)', now()); +INSERT INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (310, UUID(), 7, 'Asianux Server 4 (64 bit)', now()); -- asianux4 VMWare guest os mapping -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '4.0', 'asianux4_64Guest', 310, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '4.1', 'asianux4_64Guest', 310, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.0', 'asianux4_64Guest', 310, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.1', 'asianux4_64Guest', 310, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.5', 'asianux4_64Guest', 310, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.0', 'asianux4_64Guest', 310, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'asianux4_64Guest', 310, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'asianux4_64Guest', 310, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'asianux4_64Guest', 310, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'asianux4_64Guest', 310, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'asianux4_64Guest', 310, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.0', 'asianux4_64Guest', 310, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'asianux4_64Guest', 310, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'asianux4_64Guest', 310, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'asianux4_64Guest', 310, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'asianux4_64Guest', 310, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'asianux4_64Guest', 310, now(), 0); -- Add asianux5 32 as support guest os -INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (311, UUID(), 7, 'Asianux Server 5 (32 bit)', now()); +INSERT INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (311, UUID(), 7, 'Asianux Server 5 (32 bit)', now()); -- asianux5 VMWare guest os mapping -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.0', 'asianux5Guest', 311, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'asianux5Guest', 311, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'asianux5Guest', 311, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'asianux5Guest', 311, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'asianux5Guest', 311, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'asianux5Guest', 311, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.0', 'asianux5Guest', 311, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'asianux5Guest', 311, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'asianux5Guest', 311, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'asianux5Guest', 311, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'asianux5Guest', 311, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'asianux5Guest', 311, now(), 0); -- Add asianux5 64 as support guest os -INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (312, UUID(), 7, 'Asianux Server 5 (64 bit)', now()); +INSERT INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (312, UUID(), 7, 'Asianux Server 5 (64 bit)', now()); -- asianux5 VMWare guest os mapping -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.0', 'asianux5_64Guest', 312, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'asianux5_64Guest', 312, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'asianux5_64Guest', 312, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'asianux5_64Guest', 312, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'asianux5_64Guest', 312, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'asianux5_64Guest', 312, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.0', 'asianux5_64Guest', 312, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'asianux5_64Guest', 312, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'asianux5_64Guest', 312, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'asianux5_64Guest', 312, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'asianux5_64Guest', 312, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'asianux5_64Guest', 312, now(), 0); -- Add asianux7 32 as support guest os -INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (313, UUID(), 7, 'Asianux Server 7 (32 bit)', now()); +INSERT INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (313, UUID(), 7, 'Asianux Server 7 (32 bit)', now()); -- asianux7 VMWare guest os mapping -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'asianux7Guest', 313, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'asianux7Guest', 313, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'asianux7Guest', 313, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'asianux7Guest', 313, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'asianux7Guest', 313, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'asianux7Guest', 313, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'asianux7Guest', 313, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'asianux7Guest', 313, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'asianux7Guest', 313, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'asianux7Guest', 313, now(), 0); -- Add asianux7 64 as support guest os -INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (314, UUID(), 7, 'Asianux Server 7 (64 bit)', now()); +INSERT INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (314, UUID(), 7, 'Asianux Server 7 (64 bit)', now()); -- asianux7 VMWare guest os mapping -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'asianux7_64Guest', 314, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'asianux7_64Guest', 314, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'asianux7_64Guest', 314, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'asianux7_64Guest', 314, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'asianux7_64Guest', 314, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'asianux7_64Guest', 314, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'asianux7_64Guest', 314, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'asianux7_64Guest', 314, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'asianux7_64Guest', 314, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'asianux7_64Guest', 314, now(), 0); -- Add asianux8 as support guest os -INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (315, UUID(), 7, 'Asianux Server 8 (64 bit)', now()); +INSERT INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (315, UUID(), 7, 'Asianux Server 8 (64 bit)', now()); -- asianux8 VMWare guest os mapping -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'asianux8_64Guest', 315, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'asianux8_64Guest', 315, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'asianux8_64Guest', 315, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'asianux8_64Guest', 315, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'asianux8_64Guest', 315, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'asianux8_64Guest', 315, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'asianux8_64Guest', 315, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'asianux8_64Guest', 315, now(), 0); -- Add eComStation 2.0 as support guest os -INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (316, UUID(), 7, 'eComStation 2.0', now()); +INSERT INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (316, UUID(), 7, 'eComStation 2.0', now()); -- eComStation 2.0 VMWare guest os mapping -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'eComStation2Guest', 316, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'eComStation2Guest', 316, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'eComStation2Guest', 316, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'eComStation2Guest', 316, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'eComStation2Guest', 316, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'eComStation2Guest', 316, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'eComStation2Guest', 316, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'eComStation2Guest', 316, now(), 0); -- Add macOS 10.13 (64 bit) as support guest os -INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (317, UUID(), 7, 'macOS 10.13 (64 bit)', now()); +INSERT INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (317, UUID(), 7, 'macOS 10.13 (64 bit)', now()); -- macOS 10.13 (64 bit) VMWare guest os mapping -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'darwin17_64Guest', 317, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'darwin17_64Guest', 317, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'darwin17_64Guest', 317, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'darwin17_64Guest', 317, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'darwin17_64Guest', 317, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'darwin17_64Guest', 317, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'darwin17_64Guest', 317, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'darwin17_64Guest', 317, now(), 0); -- Add macOS 10.14 (64 bit) as support guest os -INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (318, UUID(), 7, 'macOS 10.14 (64 bit)', now()); +INSERT INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (318, UUID(), 7, 'macOS 10.14 (64 bit)', now()); -- macOS 10.14 (64 bit) VMWare guest os mapping -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'darwin18_64Guest', 318, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'darwin18_64Guest', 318, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'darwin18_64Guest', 318, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'darwin18_64Guest', 318, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'darwin18_64Guest', 318, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'darwin18_64Guest', 318, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'darwin18_64Guest', 318, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'darwin18_64Guest', 318, now(), 0); -- Add Fedora Linux (64 bit) as support guest os -INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (319, UUID(), 7, 'Fedora Linux (64 bit)', now()); +INSERT INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (319, UUID(), 7, 'Fedora Linux (64 bit)', now()); -- Fedora Linux (64 bit) VMWare guest os mapping -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.1', 'fedora64Guest', 319, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.5', 'fedora64Guest', 319, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.0', 'fedora64Guest', 319, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'fedora64Guest', 319, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'fedora64Guest', 319, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'fedora64Guest', 319, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'fedora64Guest', 319, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'fedora64Guest', 319, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.0', 'fedora64Guest', 319, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'fedora64Guest', 319, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'fedora64Guest', 319, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'fedora64Guest', 319, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'fedora64Guest', 319, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'fedora64Guest', 319, now(), 0); -- Add Fedora Linux as support guest os -INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (320, UUID(), 7, 'Fedora Linux', now()); +INSERT INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (320, UUID(), 7, 'Fedora Linux', now()); -- Fedora Linux VMWare guest os mapping -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.1', 'fedoraGuest', 320, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.5', 'fedoraGuest', 320, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.0', 'fedoraGuest', 320, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'fedoraGuest', 320, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'fedoraGuest', 320, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'fedoraGuest', 320, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'fedoraGuest', 320, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'fedoraGuest', 320, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.0', 'fedoraGuest', 320, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'fedoraGuest', 320, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'fedoraGuest', 320, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'fedoraGuest', 320, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'fedoraGuest', 320, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'fedoraGuest', 320, now(), 0); -- Add Mandrake Linux as support guest os -INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (321, UUID(), 7, 'Mandrake Linux', now()); +INSERT INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (321, UUID(), 7, 'Mandrake Linux', now()); -- Mandrake Linux VMWare guest os mapping -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.5', 'mandrakeGuest', 321, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.0', 'mandrakeGuest', 321, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'mandrakeGuest', 321, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'mandrakeGuest', 321, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'mandrakeGuest', 321, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'mandrakeGuest', 321, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'mandrakeGuest', 321, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.0', 'mandrakeGuest', 321, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'mandrakeGuest', 321, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'mandrakeGuest', 321, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'mandrakeGuest', 321, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'mandrakeGuest', 321, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'mandrakeGuest', 321, now(), 0); -- Add Mandriva Linux (64 bit) as support guest os -INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (322, UUID(), 7, 'Mandriva Linux (64 bit)', now()); +INSERT INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (322, UUID(), 7, 'Mandriva Linux (64 bit)', now()); -- Mandriva Linux (64 bit) VMWare guest os mapping -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '4.0', 'mandriva64Guest', 322, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '4.1', 'mandriva64Guest', 322, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.0', 'mandriva64Guest', 322, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.1', 'mandriva64Guest', 322, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.5', 'mandriva64Guest', 322, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.0', 'mandriva64Guest', 322, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'mandriva64Guest', 322, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'mandriva64Guest', 322, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'mandriva64Guest', 322, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'mandriva64Guest', 322, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'mandriva64Guest', 322, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.0', 'mandriva64Guest', 322, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'mandriva64Guest', 322, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'mandriva64Guest', 322, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'mandriva64Guest', 322, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'mandriva64Guest', 322, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'mandriva64Guest', 322, now(), 0); -- Add Mandriva Linux as support guest os -INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (323, UUID(), 7, 'Mandriva Linux', now()); +INSERT INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (323, UUID(), 7, 'Mandriva Linux', now()); -- Mandriva Linux VMWare guest os mapping -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '4.0', 'mandrivaGuest', 323, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '4.1', 'mandrivaGuest', 323, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.0', 'mandrivaGuest', 323, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.1', 'mandrivaGuest', 323, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.5', 'mandrivaGuest', 323, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.0', 'mandrivaGuest', 323, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'mandrivaGuest', 323, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'mandrivaGuest', 323, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'mandrivaGuest', 323, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'mandrivaGuest', 323, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'mandrivaGuest', 323, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.0', 'mandrivaGuest', 323, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'mandrivaGuest', 323, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'mandrivaGuest', 323, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'mandrivaGuest', 323, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'mandrivaGuest', 323, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'mandrivaGuest', 323, now(), 0); -- Add SCO OpenServer 5 as support guest os -INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (324, UUID(), 7, 'SCO OpenServer 5', now()); +INSERT INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (324, UUID(), 7, 'SCO OpenServer 5', now()); -- SCO OpenServer 5 VMWare guest os mapping -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '4.0', 'openServer5Guest', 324, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '4.1', 'openServer5Guest', 324, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.0', 'openServer5Guest', 324, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.1', 'openServer5Guest', 324, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.5', 'openServer5Guest', 324, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.0', 'openServer5Guest', 324, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'openServer5Guest', 324, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'openServer5Guest', 324, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'openServer5Guest', 324, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'openServer5Guest', 324, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'openServer5Guest', 324, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.0', 'openServer5Guest', 324, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'openServer5Guest', 324, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'openServer5Guest', 324, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'openServer5Guest', 324, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'openServer5Guest', 324, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'openServer5Guest', 324, now(), 0); -- Add SCO OpenServer 6 as support guest os -INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (325, UUID(), 7, 'SCO OpenServer 6', now()); +INSERT INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (325, UUID(), 7, 'SCO OpenServer 6', now()); -- SCO OpenServer 6 VMWare guest os mapping -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '4.0', 'openServer6Guest', 325, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '4.1', 'openServer6Guest', 325, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.0', 'openServer6Guest', 325, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.1', 'openServer6Guest', 325, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.5', 'openServer6Guest', 325, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.0', 'openServer6Guest', 325, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'openServer6Guest', 325, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'openServer6Guest', 325, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'openServer6Guest', 325, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'openServer6Guest', 325, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'openServer6Guest', 325, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.0', 'openServer6Guest', 325, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'openServer6Guest', 325, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'openServer6Guest', 325, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'openServer6Guest', 325, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'openServer6Guest', 325, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'openServer6Guest', 325, now(), 0); -- Add OpenSUSE Linux (64 bit) as support guest os -INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (326, UUID(), 7, 'OpenSUSE Linux (64 bit)', now()); +INSERT INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (326, UUID(), 7, 'OpenSUSE Linux (64 bit)', now()); -- OpenSUSE Linux (64 bit) VMWare guest os mapping -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.1', 'opensuse64Guest', 326, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.5', 'opensuse64Guest', 326, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.0', 'opensuse64Guest', 326, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'opensuse64Guest', 326, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'opensuse64Guest', 326, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'opensuse64Guest', 326, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'opensuse64Guest', 326, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'opensuse64Guest', 326, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.0', 'opensuse64Guest', 326, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'opensuse64Guest', 326, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'opensuse64Guest', 326, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'opensuse64Guest', 326, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'opensuse64Guest', 326, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'opensuse64Guest', 326, now(), 0); -- Add SCO OpenServer 6 as support guest os -INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (327, UUID(), 7, 'SCO OpenServer 6', now()); +INSERT INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (327, UUID(), 7, 'SCO OpenServer 6', now()); -- SCO OpenServer 6 VMWare guest os mapping -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.1', 'opensuseGuest', 327, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.5', 'opensuseGuest', 327, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.0', 'opensuseGuest', 327, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'opensuseGuest', 327, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'opensuseGuest', 327, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'opensuseGuest', 327, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'opensuseGuest', 327, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'opensuseGuest', 327, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.0', 'opensuseGuest', 327, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'opensuseGuest', 327, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'opensuseGuest', 327, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'opensuseGuest', 327, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'opensuseGuest', 327, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'opensuseGuest', 327, now(), 0); -- Add Solaris 11 (64 bit) as support guest os -INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (328, UUID(), 7, 'Solaris 11 (64 bit)', now()); +INSERT INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (328, UUID(), 7, 'Solaris 11 (64 bit)', now()); -- Solaris 11 (64 bit) VMWare guest os mapping -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.1', 'solaris11_64Guest', 328, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.5', 'solaris11_64Guest', 328, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.0', 'solaris11_64Guest', 328, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'solaris11_64Guest', 328, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'solaris11_64Guest', 328, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'solaris11_64Guest', 328, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'solaris11_64Guest', 328, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'solaris11_64Guest', 328, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.0', 'solaris11_64Guest', 328, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'solaris11_64Guest', 328, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'solaris11_64Guest', 328, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'solaris11_64Guest', 328, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'solaris11_64Guest', 328, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'solaris11_64Guest', 328, now(), 0); -- Add VMware Photon (64 bit) as support guest os -INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (329, UUID(), 7, 'VMware Photon (64 bit)', now()); +INSERT INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (329, UUID(), 7, 'VMware Photon (64 bit)', now()); -- VMware Photon (64 bit) VMWare guest os mapping -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'vmwarePhoton64Guest', 329, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'vmwarePhoton64Guest', 329, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'vmwarePhoton64Guest', 329, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'vmwarePhoton64Guest', 329, now(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'vmwarePhoton64Guest', 329, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.5', 'vmwarePhoton64Guest', 329, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7', 'vmwarePhoton64Guest', 329, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.1', 'vmwarePhoton64Guest', 329, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.2', 'vmwarePhoton64Guest', 329, now(), 0); +INSERT INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '6.7.3', 'vmwarePhoton64Guest', 329, now(), 0); From ce236e94ea71f3f0795fd0a2b8de8ba9000cb435 Mon Sep 17 00:00:00 2001 From: Vladimir Petrov Date: Tue, 20 Oct 2020 18:37:56 +0300 Subject: [PATCH 146/164] Smoke test for VMWare storage policies. --- test/integration/smoke/test_storage_policy.py | 263 ++++++++++++++++++ 1 file changed, 263 insertions(+) create mode 100644 test/integration/smoke/test_storage_policy.py diff --git a/test/integration/smoke/test_storage_policy.py b/test/integration/smoke/test_storage_policy.py new file mode 100644 index 00000000000..afb91ed9014 --- /dev/null +++ b/test/integration/smoke/test_storage_policy.py @@ -0,0 +1,263 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" Test for VMWare storage policies +""" +# Import Local Modules +from nose.plugins.attrib import attr +from marvin.cloudstackTestCase import cloudstackTestCase +from marvin.lib.utils import cleanup_resources +from marvin.lib.base import (Account, + VirtualMachine, + ServiceOffering, + NetworkOffering, + Network, + Volume, + DiskOffering) +from marvin.lib.common import (get_zone, + get_domain, + get_test_template) +from marvin.codes import PASS +from marvin.cloudstackAPI import (importVsphereStoragePolicies) +from marvin.cloudstackAPI import (listVsphereStoragePolicies) + + +class TestVMWareStoragePolicies(cloudstackTestCase): + + @classmethod + def setUpClass(cls): + + cls.testClient = super(TestVMWareStoragePolicies, cls).getClsTestClient() + cls.apiclient = cls.testClient.getApiClient() + + cls.testdata = cls.testClient.getParsedTestDataConfig() + # Get Zone, Domain and templates + cls.domain = get_domain(cls.apiclient) + cls.zone = get_zone(cls.apiclient, cls.testClient.getZoneForTests()) + cls._cleanup = [] + cls.hypervisor = cls.testClient.getHypervisorInfo() + cls.network_offering = NetworkOffering.create( + cls.apiclient, + cls.testdata["l2-network_offering"], + ) + cls.network_offering.update(cls.apiclient, state='Enabled') + cls.template = get_test_template( + cls.apiclient, + cls.zone.id, + cls.hypervisor, + ) + cls.cleanup = [cls.network_offering] + return + + @classmethod + def tearDownClass(cls): + try: + cleanup_resources(cls.apiclient, cls.cleanup) + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + + def setUp(self): + + self.dbclient = self.testClient.getDbConnection() + self.hypervisor = self.testClient.getHypervisorInfo() + self.testdata["virtual_machine"]["zoneid"] = self.zone.id + self.testdata["virtual_machine"]["template"] = self.template.id + self.account = Account.create( + self.apiclient, + self.testdata["account"], + domainid=self.domain.id + ) + self.cleanup = [self.account] + return + + def tearDown(self): + try: + self.debug("Cleaning up the resources") + cleanup_resources(self.apiclient, self.cleanup) + self.debug("Cleanup complete!") + except Exception as e: + self.debug("Warning! Exception in tearDown: %s" % e) + + def import_vmware_storage_policies(self, apiclient): + cmd = importVsphereStoragePolicies.importVsphereStoragePoliciesCmd() + cmd.zoneid = self.zone.id + return apiclient.importVsphereStoragePolicies(cmd) + + def list_storage_policies(self, apiclient): + cmd = listVsphereStoragePolicies.listVsphereStoragePoliciesCmd() + cmd.zoneid = self.zone.id + return apiclient.listVsphereStoragePolicies(cmd) + + def create_volume(self, apiclient): + cmd = create + @attr( + tags=[ + "advanced", + "eip", + "advancedns", + "basic", + "sg"], + required_hardware="true") + def test_01_import_storage_policies(self): + """Test VMWare storage policies + """ + + # Validate the following: + # 1. Import VMWare storage policies - the command should return non-zero result + # 2. List current VMWare storage policies - the command should return non-zero result + # 3. Create service offering with first of the imported policies + # 4. Create disk offering with first of the imported policies + # 5. Create VM using the already created service offering + # 6. Create volume using the already created disk offering + # 7. Attach this volume to our VM + # 8. Detach the volume from our VM + + self.hypervisor = self.testClient.getHypervisorInfo() + if self.hypervisor.lower() != "vmware": + self.skipTest( + "VMWare storage policies feature is not supported on %s" % + self.hypervisor.lower()) + + self.debug("Importing VMWare storage policies") + imported_policies = self.import_vmware_storage_policies(self.apiclient) + + if len(imported_policies) == 0: + self.skipTest("There are no VMWare storage policies") + + self.debug("Listing VMWare storage policies") + listed_policies = self.list_storage_policies(self.apiclient) + + self.assertNotEqual( + len(listed_policies), + 0, + "Check if list of storage policies is not zero" + ) + + self.assertEqual( + len(imported_policies), + len(listed_policies), + "Check if the number of imported policies is identical to the number of listed policies" + ) + + # Create service offering with the first storage policy from the list + service_offering = ServiceOffering.create( + self.apiclient, + self.testdata["service_offering"], + storagepolicy=listed_policies[0].id + ) + self.cleanup.append(service_offering) + + # Create disk offering with the first storage policy from the list + disk_offering = DiskOffering.create( + self.apiclient, + self.testdata["disk_offering"], + storagepolicy=listed_policies[0].id + ) + self.cleanup.append(disk_offering) + + l2_network = Network.create( + self.apiclient, + self.testdata["l2-network"], + zoneid=self.zone.id, + networkofferingid=self.network_offering.id + ) + self.cleanup.append(l2_network) + + virtual_machine = VirtualMachine.create( + self.apiclient, + self.testdata["small"], + templateid=self.template.id, + accountid=self.account.name, + domainid=self.account.domainid, + zoneid=self.zone.id, + networkids=l2_network.id, + serviceofferingid=service_offering.id, + ) + self.cleanup.append(virtual_machine) + + vms = VirtualMachine.list( + self.apiclient, + id=virtual_machine.id, + listall=True + ) + self.assertEqual( + isinstance(vms, list), + True, + "listVirtualMachines returned invalid object in response." + ) + self.assertNotEqual( + len(vms), + 0, + "listVirtualMachines returned empty list." + ) + self.debug("Deployed VM on host: %s" % vms[0].hostid) + + volume = Volume.create( + self.apiclient, + self.testdata["volume"], + account=self.account.name, + domainid=self.account.domainid, + zoneid=self.zone.id, + diskofferingid=disk_offering.id + ) + self.cleanup.append(volume) + + self.debug( + "Attaching volume (ID: %s) to VM (ID: %s)" % ( + volume.id, + virtual_machine.id + )) + virtual_machine.attach_volume(self.apiclient, volume) + + list_volume_response = Volume.list( + self.apiclient, + id=volume.id + ) + self.assertEqual( + isinstance(list_volume_response, list), + True, + "Check list response returns a valid list" + ) + self.assertNotEqual( + list_volume_response, + None, + "Check if volume exists in ListVolumes" + ) + volume = list_volume_response[0] + self.assertNotEqual( + volume.virtualmachineid, + None, + "Check if volume state (attached) is reflected" + ) + + self.debug( + "Detaching volume (ID: %s) from VM (ID: %s)" % ( + volume.id, + virtual_machine.id + )) + virtual_machine.detach_volume(self.apiclient, volume) + list_volume_response = Volume.list( + self.apiclient, + id=volume.id + ) + volume = list_volume_response[0] + self.assertEqual( + volume.virtualmachineid, + None, + "Check if volume state (detached) is reflected" + ) + return From f1f490b1514aaebc2c3cd008c4e757e7c41934fc Mon Sep 17 00:00:00 2001 From: nvazquez Date: Tue, 20 Oct 2020 19:47:27 -0300 Subject: [PATCH 147/164] Fix restore VM workflow - clone and replace existing VM instead of cloning and moving disks --- .../cloud/agent/api/to/DeployAsIsInfoTO.java | 21 +- .../com/cloud/vm/VirtualMachineProfile.java | 1 - .../hypervisor/guru/VmwareVmImplementer.java | 10 +- .../vmware/resource/VmwareResource.java | 91 ++---- .../resource/VmwareStorageProcessor.java | 14 +- .../java/com/cloud/vm/UserVmManagerImpl.java | 285 +++++++++--------- 6 files changed, 185 insertions(+), 237 deletions(-) diff --git a/api/src/main/java/com/cloud/agent/api/to/DeployAsIsInfoTO.java b/api/src/main/java/com/cloud/agent/api/to/DeployAsIsInfoTO.java index 37d481ea20a..9400de09652 100644 --- a/api/src/main/java/com/cloud/agent/api/to/DeployAsIsInfoTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/DeployAsIsInfoTO.java @@ -26,42 +26,23 @@ import java.util.Map; */ public class DeployAsIsInfoTO { - private String templatePath; - private String destStoragePool; @LogLevel(LogLevel.Log4jLevel.Off) private Map properties = new HashMap<>(); private Map nicAdapterMap = new HashMap(); - private boolean replaceVm; public DeployAsIsInfoTO() { } - public DeployAsIsInfoTO(String templatePath, String destStoragePool, Map properties, - Map nicAdapterMap, boolean replaceVm) { - this.templatePath = templatePath; - this.destStoragePool = destStoragePool; + public DeployAsIsInfoTO(Map properties, Map nicAdapterMap) { this.properties = properties; this.nicAdapterMap = nicAdapterMap; - this.replaceVm = replaceVm; - } - - public String getTemplatePath() { - return templatePath; } public Map getProperties() { return properties; } - public String getDestStoragePool() { - return destStoragePool; - } - public Map getNicAdapterMap() { return nicAdapterMap; } - - public boolean isReplaceVm() { - return replaceVm; - } } diff --git a/api/src/main/java/com/cloud/vm/VirtualMachineProfile.java b/api/src/main/java/com/cloud/vm/VirtualMachineProfile.java index be90b12e91a..c17a716666d 100644 --- a/api/src/main/java/com/cloud/vm/VirtualMachineProfile.java +++ b/api/src/main/java/com/cloud/vm/VirtualMachineProfile.java @@ -69,7 +69,6 @@ public interface VirtualMachineProfile { public static final Param BootType = new Param("BootType"); public static final Param BootIntoSetup = new Param("enterHardwareSetup"); public static final Param PreserveNics = new Param("PreserveNics"); - public static final Param ReplaceDeployAsIs = new Param("ReplaceDeployAsIs"); private String name; diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java index c5a01979105..b8de0bbe1b6 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java @@ -186,17 +186,9 @@ class VmwareVmImplementer { * Set the information relevant for deploy-as-is VMs on the VM TO */ private void setDeployAsIsInfoTO(VirtualMachineProfile vm, VirtualMachineTO to, Map details) { - String configuration = details.getOrDefault(VmDetailConstants.DEPLOY_AS_IS_CONFIGURATION, null); Map properties = deployAsIsHelper.getVirtualMachineDeployAsIsProperties(vm); - String templatePath = null; - String destStoragePool = null; - if (vm.getVirtualMachine().getState() == VirtualMachine.State.Starting) { - destStoragePool = deployAsIsHelper.getAllocatedVirtualMachineDestinationStoragePool(vm); - templatePath = deployAsIsHelper.getAllocatedVirtualMachineTemplatePath(vm, configuration, destStoragePool); - } Map nicsAdapterMapping = deployAsIsHelper.getAllocatedVirtualMachineNicsAdapterMapping(vm, to.getNics()); - boolean replaceVm = vm.getParameter(VirtualMachineProfile.Param.ReplaceDeployAsIs) != null ? (Boolean) vm.getParameter(VirtualMachineProfile.Param.ReplaceDeployAsIs) : false; - DeployAsIsInfoTO info = new DeployAsIsInfoTO(templatePath, destStoragePool, properties, nicsAdapterMapping, replaceVm); + DeployAsIsInfoTO info = new DeployAsIsInfoTO(properties, nicsAdapterMapping); to.setDeployAsIsInfo(info); } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 544d779daf2..0090833491c 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -48,7 +48,6 @@ import javax.xml.datatype.XMLGregorianCalendar; import com.cloud.agent.api.to.DataTO; import com.cloud.agent.api.to.DeployAsIsInfoTO; import com.cloud.agent.api.ValidateVcenterDetailsCommand; -import com.cloud.storage.resource.VmwareStorageLayoutType; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.storage.configdrive.ConfigDrive; import org.apache.cloudstack.storage.to.TemplateObjectTO; @@ -1739,57 +1738,6 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } } - private void restoreVirtualMachineVolumesFromTemplate(String vmInternalCSName, VirtualMachineTO vmSpec, - VmwareHypervisorHost host, VirtualMachineMO virtualMachineMO, - VmwareContext context, DatacenterMO dcMo) throws Exception { - DeployAsIsInfoTO deployAsIsInfo = vmSpec.getDeployAsIsInfo(); - if (s_logger.isInfoEnabled()) { - s_logger.info("Restoring VM " + vmInternalCSName + " volumes from template as-is"); - } - String deployAsIsTemplate = deployAsIsInfo.getTemplatePath(); - String destDatastore = deployAsIsInfo.getDestStoragePool(); - - String auxVMName = vmInternalCSName + "-aux"; - _storageProcessor.cloneVMFromTemplate(host, deployAsIsTemplate, auxVMName, destDatastore); - VirtualMachineMO auxVM = host.findVmOnHyperHost(auxVMName); - if (auxVM == null) { - s_logger.info("Cloned deploy-as-is VM " + auxVMName + " is not in this host, relocating it"); - auxVM = takeVmFromOtherHyperHost(host, auxVMName); - } - ManagedObjectReference morDatastore = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(host, destDatastore); - DatastoreMO dsMo = new DatastoreMO(context, morDatastore); - List vmdkFileBaseNames = auxVM.getVmdkFileBaseNames(); - for (String vmdkFileBaseName : vmdkFileBaseNames) { - s_logger.info("Move volume out of volume-wrapper VM " + vmdkFileBaseName); - String[] vmwareLayoutFilePair = VmwareStorageLayoutHelper.getVmdkFilePairDatastorePath(dsMo, auxVMName, vmdkFileBaseName, VmwareStorageLayoutType.VMWARE, !_fullCloneFlag); - String[] legacyCloudStackLayoutFilePair = VmwareStorageLayoutHelper.getVmdkFilePairDatastorePath(dsMo, auxVMName, vmdkFileBaseName, VmwareStorageLayoutType.CLOUDSTACK_LEGACY, !_fullCloneFlag); - for (int i=0; i[]{VirtualDisk.class}); - s_logger.info("Changing VM datastore to " + dsMo); - virtualMachineMO.changeDatastore(morDatastore); - for (String vmdkFileBaseName : vmdkFileBaseNames) { - if (dsMo.folderExists(String.format("[%s]", dsMo.getName()), vmNameInVcenter)) { - String newPath = VmwareStorageLayoutHelper.syncVolumeToVmDefaultFolder(dcMo, vmNameInVcenter, dsMo, vmdkFileBaseName, null); - s_logger.info("Attaching disk to restored VM at: " + newPath + " on datastore: " + destDatastore); - virtualMachineMO.attachDisk(new String[] {newPath}, morDatastore); - } - } - } - - private boolean isRestoreParameterSet(VirtualMachineTO vmSpec) { - DeployAsIsInfoTO deployAsIsInfo = vmSpec.getDeployAsIsInfo(); - if (deployAsIsInfo != null) { - return deployAsIsInfo.isReplaceVm(); - } - return false; - } - protected StartAnswer execute(StartCommand cmd) { if (s_logger.isInfoEnabled()) { s_logger.info("Executing resource StartCommand: " + getHumanReadableBytesJson(_gson.toJson(cmd))); @@ -1908,18 +1856,11 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } if (deployAsIs) { - if (s_logger.isTraceEnabled()) { - s_logger.trace("Deploying OVA from template as-is"); - } - String deployAsIsTemplate = deployAsIsInfo.getTemplatePath(); - String destDatastore = deployAsIsInfo.getDestStoragePool(); - _storageProcessor.cloneVMFromTemplate(hyperHost, deployAsIsTemplate, vmInternalCSName, destDatastore); vmMo = hyperHost.findVmOnHyperHost(vmInternalCSName); if (vmMo == null) { s_logger.info("Cloned deploy-as-is VM " + vmInternalCSName + " is not in this host, relocating it"); vmMo = takeVmFromOtherHyperHost(hyperHost, vmInternalCSName); } - mapSpecDisksToClonedDisks(vmMo, vmInternalCSName, specDisks); } else { DiskTO rootDisk = null; for (DiskTO vol : disks) { @@ -1970,10 +1911,9 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa throw new Exception("Failed to find the newly create or relocated VM. vmName: " + vmInternalCSName); } } - - if (deployAsIs && isRestoreParameterSet(vmSpec)) { - restoreVirtualMachineVolumesFromTemplate(vmInternalCSName, vmSpec, hyperHost, vmMo, context, dcMo); - mapSpecDisksToClonedDisks(vmMo, vmInternalCSName, specDisks); + if (deployAsIs) { + s_logger.info("Mapping VM disks to spec disks and tearing down datadisks (if any)"); + mapSpecDisksToClonedDisksAndTearDownDatadisks(vmMo, vmInternalCSName, specDisks); } int disksChanges = getDisksChangesNumberFromDisksSpec(disks, deployAsIs); @@ -2569,6 +2509,12 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } } + private void tearDownVMDisks(VirtualMachineMO vmMo, List disks) throws Exception { + for (VirtualDisk disk : disks) { + vmMo.tearDownDevice(disk); + } + } + private String getGuestOsIdFromVmSpec(VirtualMachineTO vmSpec, boolean deployAsIs) { return translateGuestOsIdentifier(vmSpec.getArch(), vmSpec.getOs(), vmSpec.getPlatformEmulator()).value(); } @@ -2624,21 +2570,18 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa /** * Modify the specDisks information to match the cloned VM's disks (from vmMo VM) */ - private void mapSpecDisksToClonedDisks(VirtualMachineMO vmMo, String vmInternalCSName, DiskTO[] specDisks) { + private void mapSpecDisksToClonedDisksAndTearDownDatadisks(VirtualMachineMO vmMo, String vmInternalCSName, DiskTO[] specDisks) { try { s_logger.debug("Mapping spec disks information to cloned VM disks for VM " + vmInternalCSName); if (vmMo != null && ArrayUtils.isNotEmpty(specDisks)) { List vmDisks = vmMo.getVirtualDisks(); - List sortedDisks = Arrays.asList(sortVolumesByDeviceId(specDisks)) + List rootDisks = new ArrayList<>(); + List sortedRootDisksFromSpec = Arrays.asList(sortVolumesByDeviceId(specDisks)) .stream() .filter(x -> x.getType() == Volume.Type.ROOT) .collect(Collectors.toList()); - if (sortedDisks.size() != vmDisks.size()) { - s_logger.error("Different number of root disks spec vs cloned deploy-as-is VM disks: " + sortedDisks.size() + " - " + vmDisks.size()); - return; - } - for (int i = 0; i < sortedDisks.size(); i++) { - DiskTO specDisk = sortedDisks.get(i); + for (int i = 0; i < sortedRootDisksFromSpec.size(); i++) { + DiskTO specDisk = sortedRootDisksFromSpec.get(i); VirtualDisk vmDisk = vmDisks.get(i); DataTO dataVolume = specDisk.getData(); if (dataVolume instanceof VolumeObjectTO) { @@ -2662,6 +2605,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } volumeObjectTO.setPath(relativePath); specDisk.setPath(relativePath); + rootDisks.add(vmDisk); } else { s_logger.error("Empty backing filename for volume " + volumeObjectTO.getName()); } @@ -2670,6 +2614,11 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } } } + vmDisks.removeAll(rootDisks); + if (CollectionUtils.isNotEmpty(vmDisks)) { + s_logger.info("Tearing down datadisks for deploy-as-is VM"); + tearDownVMDisks(vmMo, vmDisks); + } } } catch (Exception e) { String msg = "Error mapping deploy-as-is VM disks from cloned VM " + vmInternalCSName; diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java index 7127031ae9b..85aacd34f42 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -810,9 +810,21 @@ public class VmwareStorageProcessor implements StorageProcessor { DatastoreMO dsMo = new DatastoreMO(context, morDatastore); String vmdkName = volume.getName(); + String vmName = volume.getVmName(); String vmdkFileBaseName = null; if (template.isDeployAsIs() && volume.getVolumeType() == Volume.Type.ROOT) { - s_logger.info("ROOT Volume from deploy-as-is template, no need to create the volume at this point, will be cloned from template"); + VirtualMachineMO existingVm = dcMo.findVm(vmName); + if (volume.getDeviceId().equals(0L)) { + if (existingVm != null) { + s_logger.info("Found existing VM " + vmName + " before cloning from template, destroying it"); + existingVm.detachAllDisks(); + existingVm.destroy(); + } + s_logger.info("ROOT Volume from deploy-as-is template, cloning template"); + cloneVMFromTemplate(hyperHost, template.getPath(), vmName, primaryStore.getUuid()); + } else { + s_logger.info("ROOT Volume from deploy-as-is template, volume already created at this point"); + } } else { if (srcStore == null) { // create a root volume for blank VM (created from ISO) diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 227ee5abbbc..ac55e404459 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -6724,6 +6724,47 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir return _itMgr.restoreVirtualMachine(vm.getId(), newTemplateId); } + private VMTemplateVO getRestoreVirtualMachineTemplate(Account caller, Long newTemplateId, List rootVols, UserVmVO vm) { + VMTemplateVO template = null; + if (CollectionUtils.isNotEmpty(rootVols)) { + VolumeVO root = rootVols.get(0); + Long templateId = root.getTemplateId(); + boolean isISO = false; + if (templateId == null) { + // Assuming that for a vm deployed using ISO, template ID is set to NULL + isISO = true; + templateId = vm.getIsoId(); + } + //newTemplateId can be either template or ISO id. In the following snippet based on the vm deployment (from template or ISO) it is handled accordingly + if (newTemplateId != null) { + template = _templateDao.findById(newTemplateId); + _accountMgr.checkAccess(caller, null, true, template); + if (isISO) { + if (!template.getFormat().equals(ImageFormat.ISO)) { + throw new InvalidParameterValueException("Invalid ISO id provided to restore the VM "); + } + } else { + if (template.getFormat().equals(ImageFormat.ISO)) { + throw new InvalidParameterValueException("Invalid template id provided to restore the VM "); + } + } + } else { + if (isISO && templateId == null) { + throw new CloudRuntimeException("Cannot restore the VM since there is no ISO attached to VM"); + } + template = _templateDao.findById(templateId); + if (template == null) { + InvalidParameterValueException ex = new InvalidParameterValueException("Cannot find template/ISO for specified volumeid and vmId"); + ex.addProxyObject(vm.getUuid(), "vmId"); + ex.addProxyObject(root.getUuid(), "volumeId"); + throw ex; + } + } + } + + return template; + } + @Override public UserVm restoreVirtualMachine(final Account caller, final long vmId, final Long newTemplateId) throws InsufficientCapacityException, ResourceUnavailableException { Long userId = caller.getId(); @@ -6761,163 +6802,137 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir ex.addProxyObject(vm.getUuid(), "vmId"); throw ex; } - VolumeVO root = rootVols.get(0); - if ( !Volume.State.Allocated.equals(root.getState()) || newTemplateId != null ){ - Long templateId = root.getTemplateId(); - boolean isISO = false; - if (templateId == null) { - // Assuming that for a vm deployed using ISO, template ID is set to NULL - isISO = true; - templateId = vm.getIsoId(); - } - // If target VM has associated VM snapshots then don't allow restore of VM - List vmSnapshots = _vmSnapshotDao.findByVm(vmId); - if (vmSnapshots.size() > 0) { - throw new InvalidParameterValueException("Unable to restore VM, please remove VM snapshots before restoring VM"); - } + // If target VM has associated VM snapshots then don't allow restore of VM + List vmSnapshots = _vmSnapshotDao.findByVm(vmId); + if (vmSnapshots.size() > 0) { + throw new InvalidParameterValueException("Unable to restore VM, please remove VM snapshots before restoring VM"); + } - VMTemplateVO template = null; - //newTemplateId can be either template or ISO id. In the following snippet based on the vm deployment (from template or ISO) it is handled accordingly - if (newTemplateId != null) { - template = _templateDao.findById(newTemplateId); - _accountMgr.checkAccess(caller, null, true, template); - if (isISO) { - if (!template.getFormat().equals(ImageFormat.ISO)) { - throw new InvalidParameterValueException("Invalid ISO id provided to restore the VM "); + VMTemplateVO template = getRestoreVirtualMachineTemplate(caller, newTemplateId, rootVols, vm); + checkRestoreVmFromTemplate(vm, template); + + if (needRestart) { + try { + _itMgr.stop(vm.getUuid()); + } catch (ResourceUnavailableException e) { + s_logger.debug("Stop vm " + vm.getUuid() + " failed", e); + CloudRuntimeException ex = new CloudRuntimeException("Stop vm failed for specified vmId"); + ex.addProxyObject(vm.getUuid(), "vmId"); + throw ex; + } + } + + List newVols = new ArrayList<>(); + for (VolumeVO root : rootVols) { + if ( !Volume.State.Allocated.equals(root.getState()) || newTemplateId != null ){ + Long templateId = root.getTemplateId(); + boolean isISO = false; + if (templateId == null) { + // Assuming that for a vm deployed using ISO, template ID is set to NULL + isISO = true; + templateId = vm.getIsoId(); + } + + /* If new template/ISO is provided allocate a new volume from new template/ISO otherwise allocate new volume from original template/ISO */ + Volume newVol = null; + if (newTemplateId != null) { + if (isISO) { + newVol = volumeMgr.allocateDuplicateVolume(root, null); + vm.setIsoId(newTemplateId); + vm.setGuestOSId(template.getGuestOSId()); + vm.setTemplateId(newTemplateId); + _vmDao.update(vmId, vm); + } else { + newVol = volumeMgr.allocateDuplicateVolume(root, newTemplateId); + vm.setGuestOSId(template.getGuestOSId()); + vm.setTemplateId(newTemplateId); + _vmDao.update(vmId, vm); } } else { - if (template.getFormat().equals(ImageFormat.ISO)) { - throw new InvalidParameterValueException("Invalid template id provided to restore the VM "); - } - } - } else { - if (isISO && templateId == null) { - throw new CloudRuntimeException("Cannot restore the VM since there is no ISO attached to VM"); - } - template = _templateDao.findById(templateId); - if (template == null) { - InvalidParameterValueException ex = new InvalidParameterValueException("Cannot find template/ISO for specified volumeid and vmId"); - ex.addProxyObject(vm.getUuid(), "vmId"); - ex.addProxyObject(root.getUuid(), "volumeId"); - throw ex; - } - } - - checkRestoreVmFromTemplate(vm, template); - - if (needRestart) { - try { - _itMgr.stop(vm.getUuid()); - } catch (ResourceUnavailableException e) { - s_logger.debug("Stop vm " + vm.getUuid() + " failed", e); - CloudRuntimeException ex = new CloudRuntimeException("Stop vm failed for specified vmId"); - ex.addProxyObject(vm.getUuid(), "vmId"); - throw ex; - } - } - - /* If new template/ISO is provided allocate a new volume from new template/ISO otherwise allocate new volume from original template/ISO */ - Volume newVol = null; - if (newTemplateId != null) { - if (isISO) { newVol = volumeMgr.allocateDuplicateVolume(root, null); - vm.setIsoId(newTemplateId); - vm.setGuestOSId(template.getGuestOSId()); - vm.setTemplateId(newTemplateId); - _vmDao.update(vmId, vm); - } else { - newVol = volumeMgr.allocateDuplicateVolume(root, newTemplateId); - vm.setGuestOSId(template.getGuestOSId()); - vm.setTemplateId(newTemplateId); - _vmDao.update(vmId, vm); } - } else { - newVol = volumeMgr.allocateDuplicateVolume(root, null); - } + newVols.add(newVol); - // 1. Save usage event and update resource count for user vm volumes - _resourceLimitMgr.incrementResourceCount(newVol.getAccountId(), ResourceType.volume, newVol.isDisplay()); - _resourceLimitMgr.incrementResourceCount(newVol.getAccountId(), ResourceType.primary_storage, newVol.isDisplay(), new Long(newVol.getSize())); - // 2. Create Usage event for the newly created volume - UsageEventVO usageEvent = new UsageEventVO(EventTypes.EVENT_VOLUME_CREATE, newVol.getAccountId(), newVol.getDataCenterId(), newVol.getId(), newVol.getName(), newVol.getDiskOfferingId(), template.getId(), newVol.getSize()); - _usageEventDao.persist(usageEvent); + // 1. Save usage event and update resource count for user vm volumes + _resourceLimitMgr.incrementResourceCount(newVol.getAccountId(), ResourceType.volume, newVol.isDisplay()); + _resourceLimitMgr.incrementResourceCount(newVol.getAccountId(), ResourceType.primary_storage, newVol.isDisplay(), new Long(newVol.getSize())); + // 2. Create Usage event for the newly created volume + UsageEventVO usageEvent = new UsageEventVO(EventTypes.EVENT_VOLUME_CREATE, newVol.getAccountId(), newVol.getDataCenterId(), newVol.getId(), newVol.getName(), newVol.getDiskOfferingId(), template.getId(), newVol.getSize()); + _usageEventDao.persist(usageEvent); - handleManagedStorage(vm, root); + handleManagedStorage(vm, root); - _volsDao.attachVolume(newVol.getId(), vmId, newVol.getDeviceId()); + _volsDao.attachVolume(newVol.getId(), vmId, newVol.getDeviceId()); - // Detach, destroy and create the usage event for the old root volume. - _volsDao.detachVolume(root.getId()); - volumeMgr.destroyVolume(root); + // Detach, destroy and create the usage event for the old root volume. + _volsDao.detachVolume(root.getId()); + volumeMgr.destroyVolume(root); - // For VMware hypervisor since the old root volume is replaced by the new root volume, force expunge old root volume if it has been created in storage - if (vm.getHypervisorType() == HypervisorType.VMware) { - VolumeInfo volumeInStorage = volFactory.getVolume(root.getId()); - if (volumeInStorage != null) { - s_logger.info("Expunging volume " + root.getId() + " from primary data store"); - AsyncCallFuture future = _volService.expungeVolumeAsync(volFactory.getVolume(root.getId())); - try { - future.get(); - } catch (Exception e) { - s_logger.debug("Failed to expunge volume:" + root.getId(), e); - } - } - } - - Map params = null; - String password = null; - - if (template.isEnablePassword()) { - password = _mgr.generateRandomPassword(); - boolean result = resetVMPasswordInternal(vmId, password); - if (!result) { - throw new CloudRuntimeException("VM reset is completed but failed to reset password for the virtual machine "); - } - vm.setPassword(password); - } - - if (needRestart) { - try { - if (vm.getDetail(VmDetailConstants.PASSWORD) != null) { - params = new HashMap<>(); - params.put(VirtualMachineProfile.Param.VmPassword, password); - } - if (template.isDeployAsIs()) { - if (params == null) { - params = new HashMap<>(); - } - params.put(VirtualMachineProfile.Param.ReplaceDeployAsIs, true); - } - _itMgr.start(vm.getUuid(), params); - vm = _vmDao.findById(vmId); - if (template.isEnablePassword()) { - // this value is not being sent to the backend; need only for api - // display purposes - vm.setPassword(password); - if (vm.isUpdateParameters()) { - vm.setUpdateParameters(false); - _vmDao.loadDetails(vm); - if (vm.getDetail(VmDetailConstants.PASSWORD) != null) { - userVmDetailsDao.removeDetail(vm.getId(), VmDetailConstants.PASSWORD); - } - _vmDao.update(vm.getId(), vm); + // For VMware hypervisor since the old root volume is replaced by the new root volume, force expunge old root volume if it has been created in storage + if (vm.getHypervisorType() == HypervisorType.VMware) { + VolumeInfo volumeInStorage = volFactory.getVolume(root.getId()); + if (volumeInStorage != null) { + s_logger.info("Expunging volume " + root.getId() + " from primary data store"); + AsyncCallFuture future = _volService.expungeVolumeAsync(volFactory.getVolume(root.getId())); + try { + future.get(); + } catch (Exception e) { + s_logger.debug("Failed to expunge volume:" + root.getId(), e); } } - } catch (Exception e) { - s_logger.debug("Unable to start VM " + vm.getUuid(), e); - CloudRuntimeException ex = new CloudRuntimeException("Unable to start VM with specified id" + e.getMessage()); - ex.addProxyObject(vm.getUuid(), "vmId"); - throw ex; } } } + Map params = null; + String password = null; + + if (template.isEnablePassword()) { + password = _mgr.generateRandomPassword(); + boolean result = resetVMPasswordInternal(vmId, password); + if (!result) { + throw new CloudRuntimeException("VM reset is completed but failed to reset password for the virtual machine "); + } + vm.setPassword(password); + } + if (needRestart) { + try { + if (vm.getDetail(VmDetailConstants.PASSWORD) != null) { + params = new HashMap<>(); + params.put(VirtualMachineProfile.Param.VmPassword, password); + } + _itMgr.start(vm.getUuid(), params); + vm = _vmDao.findById(vmId); + if (template.isEnablePassword()) { + // this value is not being sent to the backend; need only for api + // display purposes + vm.setPassword(password); + if (vm.isUpdateParameters()) { + vm.setUpdateParameters(false); + _vmDao.loadDetails(vm); + if (vm.getDetail(VmDetailConstants.PASSWORD) != null) { + userVmDetailsDao.removeDetail(vm.getId(), VmDetailConstants.PASSWORD); + } + _vmDao.update(vm.getId(), vm); + } + } + } catch (Exception e) { + s_logger.debug("Unable to start VM " + vm.getUuid(), e); + CloudRuntimeException ex = new CloudRuntimeException("Unable to start VM with specified id" + e.getMessage()); + ex.addProxyObject(vm.getUuid(), "vmId"); + throw ex; + } + } + s_logger.debug("Restore VM " + vmId + " done successfully"); return vm; } + private void recreateDeployAsIsVirtualMachine(UserVmVO vm, List newVols) { + } + /** * Perform basic checkings to make sure restore is possible. If not, #InvalidParameterValueException is thrown. * From 68d51cb333eeb303e9fa02a8448e34628812867f Mon Sep 17 00:00:00 2001 From: nvazquez Date: Tue, 20 Oct 2020 19:57:42 -0300 Subject: [PATCH 148/164] Remove unused code --- server/src/main/java/com/cloud/vm/UserVmManagerImpl.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index ac55e404459..a931159ea21 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -6930,9 +6930,6 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir } - private void recreateDeployAsIsVirtualMachine(UserVmVO vm, List newVols) { - } - /** * Perform basic checkings to make sure restore is possible. If not, #InvalidParameterValueException is thrown. * From 01013302133ea99f8576299faabb5240f6ac0525 Mon Sep 17 00:00:00 2001 From: dahn Date: Wed, 21 Oct 2020 09:26:30 +0200 Subject: [PATCH 149/164] class object cleanup and append --- test/integration/smoke/test_storage_policy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/smoke/test_storage_policy.py b/test/integration/smoke/test_storage_policy.py index afb91ed9014..0952187416f 100644 --- a/test/integration/smoke/test_storage_policy.py +++ b/test/integration/smoke/test_storage_policy.py @@ -60,7 +60,7 @@ class TestVMWareStoragePolicies(cloudstackTestCase): cls.zone.id, cls.hypervisor, ) - cls.cleanup = [cls.network_offering] + cls._cleanup.append(cls.network_offering) return @classmethod From 95816499d8be83b2436fbe8a9e41b2c3d0189b78 Mon Sep 17 00:00:00 2001 From: dahn Date: Wed, 21 Oct 2020 09:27:43 +0200 Subject: [PATCH 150/164] cleanup the right object list in the right order --- test/integration/smoke/test_storage_policy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/smoke/test_storage_policy.py b/test/integration/smoke/test_storage_policy.py index 0952187416f..48982446dfc 100644 --- a/test/integration/smoke/test_storage_policy.py +++ b/test/integration/smoke/test_storage_policy.py @@ -66,7 +66,7 @@ class TestVMWareStoragePolicies(cloudstackTestCase): @classmethod def tearDownClass(cls): try: - cleanup_resources(cls.apiclient, cls.cleanup) + cleanup_resources(cls.apiclient, reverved(cls._cleanup)) except Exception as e: raise Exception("Warning: Exception during cleanup : %s" % e) From 6db6660a0c9bb69a3297a83c1c402ac17a4fc0cb Mon Sep 17 00:00:00 2001 From: dahn Date: Wed, 21 Oct 2020 09:29:07 +0200 Subject: [PATCH 151/164] object creation seperated from filling it --- test/integration/smoke/test_storage_policy.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/integration/smoke/test_storage_policy.py b/test/integration/smoke/test_storage_policy.py index 48982446dfc..c89c14d5d17 100644 --- a/test/integration/smoke/test_storage_policy.py +++ b/test/integration/smoke/test_storage_policy.py @@ -81,7 +81,8 @@ class TestVMWareStoragePolicies(cloudstackTestCase): self.testdata["account"], domainid=self.domain.id ) - self.cleanup = [self.account] + self.cleanup = [] + self.cleanup.append(self.account) return def tearDown(self): From 6cde21048a0fdca87890f248b50d302be755d0c2 Mon Sep 17 00:00:00 2001 From: dahn Date: Wed, 21 Oct 2020 09:30:26 +0200 Subject: [PATCH 152/164] make sure dependencies get cleaned up after their depenedants --- test/integration/smoke/test_storage_policy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/smoke/test_storage_policy.py b/test/integration/smoke/test_storage_policy.py index c89c14d5d17..c3e04ea3d7e 100644 --- a/test/integration/smoke/test_storage_policy.py +++ b/test/integration/smoke/test_storage_policy.py @@ -88,7 +88,7 @@ class TestVMWareStoragePolicies(cloudstackTestCase): def tearDown(self): try: self.debug("Cleaning up the resources") - cleanup_resources(self.apiclient, self.cleanup) + cleanup_resources(self.apiclient, reversed(self.cleanup)) self.debug("Cleanup complete!") except Exception as e: self.debug("Warning! Exception in tearDown: %s" % e) From ff197b2b56ff8280730e5079f4c7272e5bf78b78 Mon Sep 17 00:00:00 2001 From: dahn Date: Wed, 21 Oct 2020 09:31:29 +0200 Subject: [PATCH 153/164] readability/formatting --- test/integration/smoke/test_storage_policy.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/integration/smoke/test_storage_policy.py b/test/integration/smoke/test_storage_policy.py index c3e04ea3d7e..c006e3c6c56 100644 --- a/test/integration/smoke/test_storage_policy.py +++ b/test/integration/smoke/test_storage_policy.py @@ -105,6 +105,7 @@ class TestVMWareStoragePolicies(cloudstackTestCase): def create_volume(self, apiclient): cmd = create + @attr( tags=[ "advanced", From 8c68843142863735a9e868de8496f9c63ea66db7 Mon Sep 17 00:00:00 2001 From: nvazquez Date: Wed, 21 Oct 2020 11:46:17 -0300 Subject: [PATCH 154/164] Fix boot into hardware --- .../vmware/resource/VmwareResource.java | 41 +++++++++---------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 0090833491c..a3da897b250 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -2331,29 +2331,28 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa if (deployAsIs) { setDeployAsIsProperties(vmMo, deployAsIsInfo, vmConfigSpec); - configureVNC(vmSpec, extraOptions, vmConfigSpec, hyperHost, vmInternalCSName); - } else { - configNvpExtraOption(extraOptions, vmSpec, nicUuidToDvSwitchUuid); - configCustomExtraOption(extraOptions, vmSpec); - - // config for NCC - VirtualMachine.Type vmType = cmd.getVirtualMachine().getType(); - if (vmType.equals(VirtualMachine.Type.NetScalerVm)) { - NicTO mgmtNic = vmSpec.getNics()[0]; - OptionValue option = new OptionValue(); - option.setKey("machine.id"); - option.setValue("ip=" + mgmtNic.getIp() + "&netmask=" + mgmtNic.getNetmask() + "&gateway=" + mgmtNic.getGateway()); - extraOptions.add(option); - } - - configureVNC(vmSpec, extraOptions, vmConfigSpec, hyperHost, vmInternalCSName); - - // config video card - configureVideoCard(vmMo, vmSpec, vmConfigSpec); - - setBootOptions(vmSpec, bootMode, vmConfigSpec); } + configNvpExtraOption(extraOptions, vmSpec, nicUuidToDvSwitchUuid); + configCustomExtraOption(extraOptions, vmSpec); + + // config for NCC + VirtualMachine.Type vmType = cmd.getVirtualMachine().getType(); + if (vmType.equals(VirtualMachine.Type.NetScalerVm)) { + NicTO mgmtNic = vmSpec.getNics()[0]; + OptionValue option = new OptionValue(); + option.setKey("machine.id"); + option.setValue("ip=" + mgmtNic.getIp() + "&netmask=" + mgmtNic.getNetmask() + "&gateway=" + mgmtNic.getGateway()); + extraOptions.add(option); + } + + configureVNC(vmSpec, extraOptions, vmConfigSpec, hyperHost, vmInternalCSName); + + // config video card + configureVideoCard(vmMo, vmSpec, vmConfigSpec); + + setBootOptions(vmSpec, bootMode, vmConfigSpec); + if (!StringUtils.isEmpty(vmStoragePolicyId)) { vmConfigSpec.getVmProfile().add(vmProfileSpec); if (s_logger.isTraceEnabled()) { From 5d24e735f5d2ab699ed679cd1a9a9b317c5bb7ac Mon Sep 17 00:00:00 2001 From: nvazquez Date: Thu, 22 Oct 2020 01:26:54 -0300 Subject: [PATCH 155/164] Fix smoke tests passing osTypeId parameter on VMware templates registration --- tools/marvin/marvin/lib/base.py | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/tools/marvin/marvin/lib/base.py b/tools/marvin/marvin/lib/base.py index a53e295c974..28a89df37fe 100755 --- a/tools/marvin/marvin/lib/base.py +++ b/tools/marvin/marvin/lib/base.py @@ -1370,22 +1370,24 @@ class Template: elif "hypervisor" in services: cmd.hypervisor = services["hypervisor"] - if "ostypeid" in services: - cmd.ostypeid = services["ostypeid"] - elif "ostype" in services: - # Find OSTypeId from Os type - sub_cmd = listOsTypes.listOsTypesCmd() - sub_cmd.description = services["ostype"] - ostypes = apiclient.listOsTypes(sub_cmd) + if cmd.hypervisor.lower() not in ["vmware"]: + # Since version 4.15 VMware templates honour the guest OS defined in the template + if "ostypeid" in services: + cmd.ostypeid = services["ostypeid"] + elif "ostype" in services: + # Find OSTypeId from Os type + sub_cmd = listOsTypes.listOsTypesCmd() + sub_cmd.description = services["ostype"] + ostypes = apiclient.listOsTypes(sub_cmd) - if not isinstance(ostypes, list): + if not isinstance(ostypes, list): + raise Exception( + "Unable to find Ostype id with desc: %s" % + services["ostype"]) + cmd.ostypeid = ostypes[0].id + else: raise Exception( - "Unable to find Ostype id with desc: %s" % - services["ostype"]) - cmd.ostypeid = ostypes[0].id - else: - raise Exception( - "Unable to find Ostype is required for registering template") + "Unable to find Ostype is required for registering template") cmd.url = services["url"] @@ -1421,9 +1423,6 @@ class Template: if "directdownload" in services: cmd.directdownload = services["directdownload"] - if "deployasis" in services: - cmd.deployasis = services["deployasis"] - # Register Template template = apiclient.registerTemplate(cmd) From 187813a20904fa363519374cef66ebb7c8ce3c61 Mon Sep 17 00:00:00 2001 From: nvazquez Date: Thu, 22 Oct 2020 17:14:26 -0300 Subject: [PATCH 156/164] Fixes after smoke tests results on VMware --- test/integration/smoke/test_kubernetes_clusters.py | 4 ++-- test/integration/smoke/test_storage_policy.py | 2 +- test/integration/smoke/test_templates.py | 1 - 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/test/integration/smoke/test_kubernetes_clusters.py b/test/integration/smoke/test_kubernetes_clusters.py index 2b7638d5324..2867409a693 100644 --- a/test/integration/smoke/test_kubernetes_clusters.py +++ b/test/integration/smoke/test_kubernetes_clusters.py @@ -240,8 +240,8 @@ class TestKubernetesCluster(cloudstackTestCase): if validateList(templates)[0] != PASS: details = None - if hypervisor in ["vmware"] and "details" in cks_template: - details = cks_template["details"] + if hypervisor in ["vmware"]: + details = [{"keyboard": "us"}] template = Template.register(cls.apiclient, cks_template, zoneid=cls.zone.id, hypervisor=hypervisor.lower(), randomize_name=False, details=details) template.download(cls.apiclient) return template diff --git a/test/integration/smoke/test_storage_policy.py b/test/integration/smoke/test_storage_policy.py index c006e3c6c56..a59ec32b225 100644 --- a/test/integration/smoke/test_storage_policy.py +++ b/test/integration/smoke/test_storage_policy.py @@ -66,7 +66,7 @@ class TestVMWareStoragePolicies(cloudstackTestCase): @classmethod def tearDownClass(cls): try: - cleanup_resources(cls.apiclient, reverved(cls._cleanup)) + cleanup_resources(cls.apiclient, reversed(cls._cleanup)) except Exception as e: raise Exception("Warning: Exception during cleanup : %s" % e) diff --git a/test/integration/smoke/test_templates.py b/test/integration/smoke/test_templates.py index ae34f7628f4..23d950637ce 100644 --- a/test/integration/smoke/test_templates.py +++ b/test/integration/smoke/test_templates.py @@ -84,7 +84,6 @@ class TestCreateTemplateWithChecksum(cloudstackTestCase): self.test_template.displaytext = 'test sha-1' self.test_template.url = "http://dl.openvm.eu/cloudstack/macchinina/x86_64/macchinina-vmware.ova" self.test_template.format = "OVA" - self.test_template.ostypeid = self.getOsType("Other Linux (64-bit)") self.md5 = "3c23ac66bac7888dc7c972783646c644" self.sha256 = "97aaa096d419522158c54f83eb61d9242d9f6bca9166fd4030d73683d647c7e7" From 0c6900e3bd47be005108c5b494de0518b93e978b Mon Sep 17 00:00:00 2001 From: nvazquez Date: Fri, 23 Oct 2020 10:39:21 -0300 Subject: [PATCH 157/164] Do not fail if osTypeId parameter or rootdiskcontroller/nicadapter details are set on template registration --- .../user/template/RegisterTemplateCmd.java | 18 +--------------- .../cloud/template/TemplateAdapterBase.java | 21 +++++++++++++++---- 2 files changed, 18 insertions(+), 21 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java index bcb274daded..74053ee809d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java @@ -39,12 +39,10 @@ import org.apache.cloudstack.api.response.ProjectResponse; import org.apache.cloudstack.api.response.TemplateResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.cloudstack.context.CallContext; -import org.apache.commons.collections.MapUtils; import org.apache.log4j.Logger; import com.cloud.exception.ResourceAllocationException; import com.cloud.template.VirtualMachineTemplate; -import com.cloud.vm.VmDetailConstants; @APICommand(name = "registerTemplate", description = "Registers an existing template into the CloudStack cloud. ", responseObject = TemplateResponse.class, responseView = ResponseView.Restricted, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) @@ -343,21 +341,7 @@ public class RegisterTemplateCmd extends BaseCmd implements UserCmd { "Parameter directdownload is only allowed for KVM templates"); } - if (getHypervisor().equalsIgnoreCase(Hypervisor.HypervisorType.VMware.toString())) { - if (osTypeId != null) { - throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "ostypeid is not supported on VMWare, as we honour what is defined in the template"); - } - - Map templateDetails = getDetails(); - if (MapUtils.isNotEmpty(templateDetails)) { - if (templateDetails.containsKey(VmDetailConstants.ROOT_DISK_CONTROLLER)) { - throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "rootDiskController is not supported on VMWare, as we honour what is defined in the template"); - } - if (templateDetails.containsKey(VmDetailConstants.NIC_ADAPTER)) { - throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "nicAdapter is not supported on VMWare, as we honour what is defined in the template"); - } - } - } else if (osTypeId == null) { + if (!getHypervisor().equalsIgnoreCase(Hypervisor.HypervisorType.VMware.toString()) && osTypeId == null) { throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Please provide a guest OS type"); } } diff --git a/server/src/main/java/com/cloud/template/TemplateAdapterBase.java b/server/src/main/java/com/cloud/template/TemplateAdapterBase.java index 45a10a8c4b3..8e9bab30092 100644 --- a/server/src/main/java/com/cloud/template/TemplateAdapterBase.java +++ b/server/src/main/java/com/cloud/template/TemplateAdapterBase.java @@ -27,9 +27,11 @@ import com.cloud.deployasis.DeployAsIsConstants; import com.cloud.storage.upload.params.IsoUploadParams; import com.cloud.storage.upload.params.TemplateUploadParams; import com.cloud.storage.upload.params.UploadParams; +import com.cloud.vm.VmDetailConstants; import org.apache.cloudstack.api.command.user.iso.GetUploadParamsForIsoCmd; import org.apache.cloudstack.api.command.user.template.GetUploadParamsForTemplateCmd; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.BooleanUtils; import org.apache.log4j.Logger; @@ -170,9 +172,7 @@ public abstract class TemplateAdapterBase extends AdapterBase implements Templat } if (deployAsIs) { GuestOS deployAsIsGuestOs = ApiDBUtils.findGuestOSByDisplayName(DeployAsIsConstants.DEFAULT_GUEST_OS_DEPLOY_AS_IS); - if (s_logger.isDebugEnabled()) { - s_logger.debug("Setting default guest OS for deploy-as-is template while the template registration is not completed"); - } + s_logger.info("Setting default guest OS for deploy-as-is template while the template registration is not completed"); guestOSId = deployAsIsGuestOs.getId(); } } @@ -291,9 +291,22 @@ public abstract class TemplateAdapterBase extends AdapterBase implements Templat + EnumUtils.listValues(HypervisorType.values()).replace("None, ", "")); } + Map details = cmd.getDetails(); + if (hypervisorType == HypervisorType.VMware) { + if (MapUtils.isNotEmpty(details)) { + if (details.containsKey(VmDetailConstants.ROOT_DISK_CONTROLLER)) { + s_logger.info("Ignoring the rootDiskController detail provided, as we honour what is defined in the template"); + details.remove(VmDetailConstants.ROOT_DISK_CONTROLLER); + } + if (details.containsKey(VmDetailConstants.NIC_ADAPTER)) { + s_logger.info("Ignoring the nicAdapter detail provided, as we honour what is defined in the template"); + details.remove(VmDetailConstants.NIC_ADAPTER); + } + } + } return prepare(false, CallContext.current().getCallingUserId(), cmd.getTemplateName(), cmd.getDisplayText(), cmd.getBits(), cmd.isPasswordEnabled(), cmd.getRequiresHvm(), cmd.getUrl(), cmd.isPublic(), cmd.isFeatured(), cmd.isExtractable(), cmd.getFormat(), cmd.getOsTypeId(), zoneId, hypervisorType, cmd.getChecksum(), true, - cmd.getTemplateTag(), owner, cmd.getDetails(), cmd.isSshKeyEnabled(), null, cmd.isDynamicallyScalable(), isRouting ? TemplateType.ROUTING : TemplateType.USER, + cmd.getTemplateTag(), owner, details, cmd.isSshKeyEnabled(), null, cmd.isDynamicallyScalable(), isRouting ? TemplateType.ROUTING : TemplateType.USER, cmd.isDirectDownload(), cmd.isDeployAsIs()); } From 0d7c582e22d12e2bcca637967d5de201211248e4 Mon Sep 17 00:00:00 2001 From: nvazquez Date: Sat, 24 Oct 2020 12:59:59 -0300 Subject: [PATCH 158/164] Fix macchinina checksums for VMware --- test/integration/smoke/test_templates.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/integration/smoke/test_templates.py b/test/integration/smoke/test_templates.py index 23d950637ce..dd724b8171c 100644 --- a/test/integration/smoke/test_templates.py +++ b/test/integration/smoke/test_templates.py @@ -77,15 +77,15 @@ class TestCreateTemplateWithChecksum(cloudstackTestCase): if "vmware" in self.hypervisor.lower(): self.test_template = registerTemplate.registerTemplateCmd() self.test_template = registerTemplate.registerTemplateCmd() - self.test_template.checksum = "{SHA-1}" + "178639bd5ec089a27f6d39025be28c3de5d9393b" + self.test_template.checksum = "{SHA-1}" + "3c00872599c6e1e46a358aac51080db88266cf5c" self.test_template.hypervisor = self.hypervisor self.test_template.zoneid = self.zone.id self.test_template.name = 'test sha-2333' self.test_template.displaytext = 'test sha-1' self.test_template.url = "http://dl.openvm.eu/cloudstack/macchinina/x86_64/macchinina-vmware.ova" self.test_template.format = "OVA" - self.md5 = "3c23ac66bac7888dc7c972783646c644" - self.sha256 = "97aaa096d419522158c54f83eb61d9242d9f6bca9166fd4030d73683d647c7e7" + self.md5 = "27f3c56a8c7ec7b2f3ff2199f7078006" + self.sha256 = "a7b04c1eb507f3f5de844bda352df1ea5e20335b465409493ca6ae07dfd0a158" if "xen" in self.hypervisor.lower(): self.test_template = registerTemplate.registerTemplateCmd() From a831ad125276208119379673f0ace7fbe3287d3c Mon Sep 17 00:00:00 2001 From: nvazquez Date: Sun, 25 Oct 2020 00:33:46 -0300 Subject: [PATCH 159/164] Fix detach volume on stopped Vms --- .../cloud/storage/VolumeApiServiceImpl.java | 44 ++++++++++--------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index 58e1bffaf37..ad3dd30ab8d 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -2054,31 +2054,35 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic if (!sendCommand || (answer != null && answer.getResult())) { // Mark the volume as detached _volsDao.detachVolume(volume.getId()); - String datastoreName = answer.getContextParam("datastoreName"); - if (datastoreName != null) { - StoragePoolVO storagePoolVO = _storagePoolDao.findByUuid(datastoreName); - if (storagePoolVO != null) { + + if (answer != null) { + String datastoreName = answer.getContextParam("datastoreName"); + if (datastoreName != null) { + StoragePoolVO storagePoolVO = _storagePoolDao.findByUuid(datastoreName); + if (storagePoolVO != null) { + VolumeVO volumeVO = _volsDao.findById(volumeId); + volumeVO.setPoolId(storagePoolVO.getId()); + _volsDao.update(volumeVO.getId(), volumeVO); + } else { + s_logger.warn(String.format("Unable to find datastore %s while updating the new datastore of the volume %d", datastoreName, volumeId)); + } + } + + String volumePath = answer.getContextParam("volumePath"); + if (volumePath != null) { VolumeVO volumeVO = _volsDao.findById(volumeId); - volumeVO.setPoolId(storagePoolVO.getId()); + volumeVO.setPath(volumePath); + _volsDao.update(volumeVO.getId(), volumeVO); + } + + String chainInfo = answer.getContextParam("chainInfo"); + if (chainInfo != null) { + VolumeVO volumeVO = _volsDao.findById(volumeId); + volumeVO.setChainInfo(chainInfo); _volsDao.update(volumeVO.getId(), volumeVO); - } else { - s_logger.warn(String.format("Unable to find datastore %s while updating the new datastore of the volume %d", datastoreName, volumeId)); } } - String volumePath = answer.getContextParam("volumePath"); - if (volumePath != null) { - VolumeVO volumeVO = _volsDao.findById(volumeId); - volumeVO.setPath(volumePath); - _volsDao.update(volumeVO.getId(), volumeVO); - } - - String chainInfo = answer.getContextParam("chainInfo"); - if (chainInfo != null) { - VolumeVO volumeVO = _volsDao.findById(volumeId); - volumeVO.setChainInfo(chainInfo); - _volsDao.update(volumeVO.getId(), volumeVO); - } // volume.getPoolId() should be null if the VM we are detaching the disk from has never been started before if (volume.getPoolId() != null) { DataStore dataStore = dataStoreMgr.getDataStore(volume.getPoolId(), DataStoreRole.Primary); From c405e5dc31242e64fee2fd70b1df7f3f7209405a Mon Sep 17 00:00:00 2001 From: nvazquez Date: Sun, 25 Oct 2020 02:19:22 -0300 Subject: [PATCH 160/164] Add new API to check storage policy compatible pools and fix marvin test for storage policies --- .../vmware/VmwareDatacenterService.java | 3 + .../vmware/manager/VmwareManagerImpl.java | 43 ++++++++ ...sphereStoragePolicyCompatiblePoolsCmd.java | 97 +++++++++++++++++++ test/integration/smoke/test_storage_policy.py | 28 +++++- 4 files changed, 166 insertions(+), 5 deletions(-) create mode 100644 plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVsphereStoragePolicyCompatiblePoolsCmd.java diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenterService.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenterService.java index 2e3f98d795a..431526a2c51 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenterService.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenterService.java @@ -20,12 +20,14 @@ package com.cloud.hypervisor.vmware; import com.cloud.dc.VsphereStoragePolicy; import com.cloud.exception.DiscoveryException; import com.cloud.exception.ResourceInUseException; +import com.cloud.storage.StoragePool; import com.cloud.utils.component.PluggableService; import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.api.command.admin.zone.AddVmwareDcCmd; import org.apache.cloudstack.api.command.admin.zone.ImportVsphereStoragePoliciesCmd; import org.apache.cloudstack.api.command.admin.zone.ListVmwareDcsCmd; import org.apache.cloudstack.api.command.admin.zone.ListVsphereStoragePoliciesCmd; +import org.apache.cloudstack.api.command.admin.zone.ListVsphereStoragePolicyCompatiblePoolsCmd; import org.apache.cloudstack.api.command.admin.zone.RemoveVmwareDcCmd; import org.apache.cloudstack.api.command.admin.zone.UpdateVmwareDcCmd; @@ -45,4 +47,5 @@ public interface VmwareDatacenterService extends PluggableService { List listVsphereStoragePolicies(ListVsphereStoragePoliciesCmd cmd); + List listVsphereStoragePolicyCompatibleStoragePools(ListVsphereStoragePolicyCompatiblePoolsCmd cmd); } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java index a751abf1b8e..b01f7989b6b 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java @@ -25,6 +25,7 @@ import com.cloud.agent.api.Answer; import com.cloud.agent.api.Command; import com.cloud.agent.api.StartupCommand; import com.cloud.agent.api.StartupRoutingCommand; +import com.cloud.agent.api.to.StorageFilerTO; import com.cloud.api.query.dao.TemplateJoinDao; import com.cloud.cluster.ClusterManager; import com.cloud.cluster.dao.ManagementServerHostPeerDao; @@ -41,9 +42,11 @@ import com.cloud.dc.dao.DataCenterDao; import com.cloud.dc.dao.VsphereStoragePolicyDao; import com.cloud.event.ActionEvent; import com.cloud.event.EventTypes; +import com.cloud.exception.AgentUnavailableException; import com.cloud.exception.DiscoveredWithErrorException; import com.cloud.exception.DiscoveryException; import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.OperationTimedoutException; import com.cloud.exception.ResourceInUseException; import com.cloud.host.Host; import com.cloud.host.Status; @@ -51,6 +54,7 @@ import com.cloud.host.dao.HostDao; import com.cloud.host.dao.HostDetailsDao; import com.cloud.hypervisor.Hypervisor; import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.hypervisor.HypervisorGuruManager; import com.cloud.hypervisor.dao.HypervisorCapabilitiesDao; import com.cloud.hypervisor.vmware.LegacyZoneVO; import com.cloud.hypervisor.vmware.VmwareCleanupMaid; @@ -89,6 +93,8 @@ import com.cloud.storage.ImageStoreDetailsUtil; import com.cloud.storage.JavaStorageLayer; import com.cloud.storage.StorageLayer; import com.cloud.storage.StorageManager; +import com.cloud.storage.StoragePool; +import com.cloud.storage.StoragePoolStatus; import com.cloud.storage.dao.VMTemplatePoolDao; import com.cloud.template.TemplateManager; import com.cloud.utils.FileUtil; @@ -116,6 +122,7 @@ import org.apache.cloudstack.api.command.admin.zone.AddVmwareDcCmd; import org.apache.cloudstack.api.command.admin.zone.ImportVsphereStoragePoliciesCmd; import org.apache.cloudstack.api.command.admin.zone.ListVmwareDcsCmd; import org.apache.cloudstack.api.command.admin.zone.ListVsphereStoragePoliciesCmd; +import org.apache.cloudstack.api.command.admin.zone.ListVsphereStoragePolicyCompatiblePoolsCmd; import org.apache.cloudstack.api.command.admin.zone.RemoveVmwareDcCmd; import org.apache.cloudstack.api.command.admin.zone.UpdateVmwareDcCmd; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; @@ -125,7 +132,9 @@ import org.apache.cloudstack.framework.config.Configurable; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl; import org.apache.cloudstack.management.ManagementServerHost; +import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplainceCommand; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.utils.identity.ManagementServerNode; import org.apache.log4j.Logger; @@ -217,6 +226,10 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw private TemplateManager templateManager; @Inject private VsphereStoragePolicyDao vsphereStoragePolicyDao; + @Inject + private StorageManager storageManager; + @Inject + private HypervisorGuruManager hypervisorGuruManager; private String _mountParent; private StorageLayer _storage; @@ -1057,6 +1070,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw cmdList.add(ListVmwareDcsCmd.class); cmdList.add(ImportVsphereStoragePoliciesCmd.class); cmdList.add(ListVsphereStoragePoliciesCmd.class); + cmdList.add(ListVsphereStoragePolicyCompatiblePoolsCmd.class); return cmdList; } @@ -1469,6 +1483,35 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw return Collections.emptyList(); } + @Override + public List listVsphereStoragePolicyCompatibleStoragePools(ListVsphereStoragePolicyCompatiblePoolsCmd cmd) { + Long policyId = cmd.getPolicyId(); + VsphereStoragePolicyVO storagePolicy = vsphereStoragePolicyDao.findById(policyId); + if (storagePolicy == null) { + throw new CloudRuntimeException("Storage policy with ID = " + policyId + " was not found"); + } + long zoneId = storagePolicy.getZoneId(); + List poolsInZone = primaryStorageDao.listByStatusInZone(zoneId, StoragePoolStatus.Up); + List compatiblePools = new ArrayList<>(); + for (StoragePoolVO pool : poolsInZone) { + StorageFilerTO storageFilerTO = new StorageFilerTO(pool); + List hostIds = storageManager.getUpHostsInPool(pool.getId()); + Collections.shuffle(hostIds); + CheckDataStoreStoragePolicyComplainceCommand command = new CheckDataStoreStoragePolicyComplainceCommand(storagePolicy.getPolicyId(), storageFilerTO); + long targetHostId = hypervisorGuruManager.getGuruProcessedCommandTargetHost(hostIds.get(0), command); + try { + Answer answer = _agentMgr.send(targetHostId, command); + boolean result = answer != null && answer.getResult(); + if (result) { + compatiblePools.add(pool); + } + } catch (AgentUnavailableException | OperationTimedoutException e) { + s_logger.error("Could not verify if storage policy " + storagePolicy.getName() + " is compatible with storage pool " + pool.getName()); + } + } + return compatiblePools; + } + @Override public boolean hasNexusVSM(Long clusterId) { ClusterVSMMapVO vsmMapVo = null; diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVsphereStoragePolicyCompatiblePoolsCmd.java b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVsphereStoragePolicyCompatiblePoolsCmd.java new file mode 100644 index 00000000000..ee7146a3a9e --- /dev/null +++ b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVsphereStoragePolicyCompatiblePoolsCmd.java @@ -0,0 +1,97 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.zone; + +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.hypervisor.vmware.VmwareDatacenterService; +import com.cloud.storage.StoragePool; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.BaseListCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.StoragePoolResponse; +import org.apache.cloudstack.api.response.VsphereStoragePoliciesResponse; +import org.apache.cloudstack.api.response.ZoneResponse; +import org.apache.cloudstack.context.CallContext; + +import javax.inject.Inject; +import java.util.ArrayList; +import java.util.List; + +@APICommand(name = ListVsphereStoragePolicyCompatiblePoolsCmd.APINAME, description = "List storage pools compatible with a vSphere storage policy", + responseObject = StoragePoolResponse.class, + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, + authorized = {RoleType.Admin}) +public class ListVsphereStoragePolicyCompatiblePoolsCmd extends BaseListCmd { + public static final String APINAME = "listVsphereStoragePolicyCompatiblePools"; + + @Inject + public VmwareDatacenterService vmwareDatacenterService; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.UUID, entityType = ZoneResponse.class, + description = "ID of the zone") + private Long zoneId; + + @Parameter(name = ApiConstants.POLICY_ID, type = BaseCmd.CommandType.UUID, entityType = VsphereStoragePoliciesResponse.class, + description = "ID of the storage policy") + private Long policyId; + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() throws ServerApiException, ConcurrentOperationException { + List pools = vmwareDatacenterService.listVsphereStoragePolicyCompatibleStoragePools(this); + ListResponse response = new ListResponse<>(); + List poolResponses = new ArrayList<>(); + for (StoragePool pool : pools) { + StoragePoolResponse poolResponse = _responseGenerator.createStoragePoolForMigrationResponse(pool); + poolResponse.setObjectName("storagepool"); + poolResponses.add(poolResponse); + } + response.setResponses(poolResponses); + response.setResponseName(getCommandName()); + this.setResponseObject(response); + } + + @Override + public String getCommandName() { + return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX; + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccountId(); + } + + public Long getZoneId() { + return zoneId; + } + + public Long getPolicyId() { + return policyId; + } +} diff --git a/test/integration/smoke/test_storage_policy.py b/test/integration/smoke/test_storage_policy.py index a59ec32b225..14f607ac862 100644 --- a/test/integration/smoke/test_storage_policy.py +++ b/test/integration/smoke/test_storage_policy.py @@ -31,9 +31,9 @@ from marvin.lib.base import (Account, from marvin.lib.common import (get_zone, get_domain, get_test_template) -from marvin.codes import PASS -from marvin.cloudstackAPI import (importVsphereStoragePolicies) -from marvin.cloudstackAPI import (listVsphereStoragePolicies) +from marvin.cloudstackAPI import (importVsphereStoragePolicies, + listVsphereStoragePolicies, + listVsphereStoragePolicyCompatiblePools) class TestVMWareStoragePolicies(cloudstackTestCase): @@ -103,6 +103,12 @@ class TestVMWareStoragePolicies(cloudstackTestCase): cmd.zoneid = self.zone.id return apiclient.listVsphereStoragePolicies(cmd) + def list_storage_policy_compatible_pools(self, apiclient, policyid): + cmd = listVsphereStoragePolicyCompatiblePools.listVsphereStoragePolicyCompatiblePoolsCmd() + cmd.zoneid = self.zone.id + cmd.policyid = policyid + return apiclient.listVsphereStoragePolicyCompatiblePools(cmd) + def create_volume(self, apiclient): cmd = create @@ -155,11 +161,23 @@ class TestVMWareStoragePolicies(cloudstackTestCase): "Check if the number of imported policies is identical to the number of listed policies" ) + are_compatible_pools = False + selected_policy = None + for imported_policy in imported_policies: + compatible_pools = self.list_storage_policy_compatible_pools(self.apiclient, imported_policy.id) + if len(compatible_pools) > 0: + are_compatible_pools = True + selected_policy = imported_policy + break + + if not are_compatible_pools: + self.skipTest("There are no compatible storage pools with the imported policies") + # Create service offering with the first storage policy from the list service_offering = ServiceOffering.create( self.apiclient, self.testdata["service_offering"], - storagepolicy=listed_policies[0].id + storagepolicy=selected_policy.id ) self.cleanup.append(service_offering) @@ -167,7 +185,7 @@ class TestVMWareStoragePolicies(cloudstackTestCase): disk_offering = DiskOffering.create( self.apiclient, self.testdata["disk_offering"], - storagepolicy=listed_policies[0].id + storagepolicy=selected_policy.id ) self.cleanup.append(disk_offering) From 85f5512ec3bdde91792faa92e4210035bc71cbc1 Mon Sep 17 00:00:00 2001 From: nvazquez Date: Sun, 25 Oct 2020 02:53:43 -0300 Subject: [PATCH 161/164] Fix unit test failure --- .../vmware/VmwareDatacenterApiUnitTest.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/vmware/VmwareDatacenterApiUnitTest.java b/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/vmware/VmwareDatacenterApiUnitTest.java index 1249f6ce004..95387db7797 100644 --- a/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/vmware/VmwareDatacenterApiUnitTest.java +++ b/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/vmware/VmwareDatacenterApiUnitTest.java @@ -39,6 +39,7 @@ import com.cloud.exception.ResourceInUseException; import com.cloud.host.dao.HostDao; import com.cloud.host.dao.HostDetailsDao; import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.hypervisor.HypervisorGuruManager; import com.cloud.hypervisor.dao.HypervisorCapabilitiesDao; import com.cloud.hypervisor.vmware.dao.LegacyZoneDao; import com.cloud.hypervisor.vmware.dao.VmwareDatacenterDao; @@ -51,6 +52,7 @@ import com.cloud.org.Managed.ManagedState; import com.cloud.secstorage.CommandExecLogDao; import com.cloud.server.ConfigurationServer; import com.cloud.storage.ImageStoreDetailsUtil; +import com.cloud.storage.StorageManager; import com.cloud.storage.dao.VMTemplatePoolDao; import com.cloud.template.TemplateManager; import com.cloud.user.Account; @@ -492,6 +494,16 @@ public class VmwareDatacenterApiUnitTest { return Mockito.mock(VsphereStoragePolicyDao.class); } + @Bean + public StorageManager storageManager() { + return Mockito.mock(StorageManager.class); + } + + @Bean + public HypervisorGuruManager hypervisorGuruManager() { + return Mockito.mock(HypervisorGuruManager.class); + } + public static class Library implements TypeFilter { @Override From 795dc01212afb8b3f3334f54af91883f40edaab7 Mon Sep 17 00:00:00 2001 From: nvazquez Date: Sun, 25 Oct 2020 15:14:04 -0300 Subject: [PATCH 162/164] Fix for none type response error handling --- test/integration/smoke/test_storage_policy.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/integration/smoke/test_storage_policy.py b/test/integration/smoke/test_storage_policy.py index 14f607ac862..c995b9e3fa1 100644 --- a/test/integration/smoke/test_storage_policy.py +++ b/test/integration/smoke/test_storage_policy.py @@ -161,16 +161,14 @@ class TestVMWareStoragePolicies(cloudstackTestCase): "Check if the number of imported policies is identical to the number of listed policies" ) - are_compatible_pools = False selected_policy = None for imported_policy in imported_policies: compatible_pools = self.list_storage_policy_compatible_pools(self.apiclient, imported_policy.id) - if len(compatible_pools) > 0: - are_compatible_pools = True + if compatible_pools and len(compatible_pools) > 0: selected_policy = imported_policy break - if not are_compatible_pools: + if not selected_policy: self.skipTest("There are no compatible storage pools with the imported policies") # Create service offering with the first storage policy from the list From 7d7e7f794125dafbfa483766b73e1a82edfad62a Mon Sep 17 00:00:00 2001 From: nvazquez Date: Mon, 26 Oct 2020 09:07:23 -0300 Subject: [PATCH 163/164] Check there is at least a host to verify pool compatibility for a storage policy --- .../cloud/hypervisor/vmware/manager/VmwareManagerImpl.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java index b01f7989b6b..e1e0d6b24ee 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java @@ -1496,6 +1496,10 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw for (StoragePoolVO pool : poolsInZone) { StorageFilerTO storageFilerTO = new StorageFilerTO(pool); List hostIds = storageManager.getUpHostsInPool(pool.getId()); + if (CollectionUtils.isNullOrEmpty(hostIds)) { + s_logger.debug("Did not find a suitable host to verify compatibility of the pool " + pool.getName()); + continue; + } Collections.shuffle(hostIds); CheckDataStoreStoragePolicyComplainceCommand command = new CheckDataStoreStoragePolicyComplainceCommand(storagePolicy.getPolicyId(), storageFilerTO); long targetHostId = hypervisorGuruManager.getGuruProcessedCommandTargetHost(hostIds.get(0), command); From 75019442f6b484ec331622ec579de361257d083d Mon Sep 17 00:00:00 2001 From: nvazquez Date: Mon, 26 Oct 2020 09:45:01 -0300 Subject: [PATCH 164/164] Fix storage policy smoke test --- test/integration/smoke/test_storage_policy.py | 30 ------------------- 1 file changed, 30 deletions(-) diff --git a/test/integration/smoke/test_storage_policy.py b/test/integration/smoke/test_storage_policy.py index c995b9e3fa1..a7a5f8d0cba 100644 --- a/test/integration/smoke/test_storage_policy.py +++ b/test/integration/smoke/test_storage_policy.py @@ -234,13 +234,6 @@ class TestVMWareStoragePolicies(cloudstackTestCase): ) self.cleanup.append(volume) - self.debug( - "Attaching volume (ID: %s) to VM (ID: %s)" % ( - volume.id, - virtual_machine.id - )) - virtual_machine.attach_volume(self.apiclient, volume) - list_volume_response = Volume.list( self.apiclient, id=volume.id @@ -255,27 +248,4 @@ class TestVMWareStoragePolicies(cloudstackTestCase): None, "Check if volume exists in ListVolumes" ) - volume = list_volume_response[0] - self.assertNotEqual( - volume.virtualmachineid, - None, - "Check if volume state (attached) is reflected" - ) - - self.debug( - "Detaching volume (ID: %s) from VM (ID: %s)" % ( - volume.id, - virtual_machine.id - )) - virtual_machine.detach_volume(self.apiclient, volume) - list_volume_response = Volume.list( - self.apiclient, - id=volume.id - ) - volume = list_volume_response[0] - self.assertEqual( - volume.virtualmachineid, - None, - "Check if volume state (detached) is reflected" - ) return