diff --git a/agent/bindir/libvirtqemuhook.in b/agent/bindir/libvirtqemuhook.in index 598968bdb54..27e07119ccc 100755 --- a/agent/bindir/libvirtqemuhook.in +++ b/agent/bindir/libvirtqemuhook.in @@ -78,7 +78,9 @@ def handleMigrateBegin(): def executeCustomScripts(sysArgs): - createDirectoryIfNotExists(customDir, customDirPermissions) + if not os.path.exists(customDir) or not os.path.isdir(customDir): + return + scripts = getCustomScriptsFromDirectory() for scriptName in scripts: @@ -127,12 +129,6 @@ def getCustomScriptsFromDirectory(): os.listdir(customDir)), key=lambda fileName: substringAfter(fileName, '_')) -def createDirectoryIfNotExists(dir, permissions): - if not os.path.exists(dir): - logger.info('Directory %s does not exist; creating it.' % dir) - os.makedirs(dir, permissions) - - def substringAfter(s, delimiter): return s.partition(delimiter)[2] diff --git a/agent/src/main/java/com/cloud/agent/Agent.java b/agent/src/main/java/com/cloud/agent/Agent.java index 500724dd5a3..e5fbdd7a2ed 100644 --- a/agent/src/main/java/com/cloud/agent/Agent.java +++ b/agent/src/main/java/com/cloud/agent/Agent.java @@ -39,7 +39,6 @@ import java.util.concurrent.atomic.AtomicInteger; import javax.naming.ConfigurationException; -import org.apache.cloudstack.agent.directdownload.SetupDirectDownloadCertificate; import org.apache.cloudstack.agent.lb.SetupMSListAnswer; import org.apache.cloudstack.agent.lb.SetupMSListCommand; import org.apache.cloudstack.ca.PostCertificateRenewalCommand; @@ -630,8 +629,6 @@ public class Agent implements HandlerFactory, IAgentControl { if (Host.Type.Routing.equals(_resource.getType())) { scheduleServicesRestartTask(); } - } else if (cmd instanceof SetupDirectDownloadCertificate) { - answer = setupDirectDownloadCertificate((SetupDirectDownloadCertificate) cmd); } else if (cmd instanceof SetupMSListCommand) { answer = setupManagementServerList((SetupMSListCommand) cmd); } else { @@ -683,31 +680,6 @@ public class Agent implements HandlerFactory, IAgentControl { } } - private Answer setupDirectDownloadCertificate(SetupDirectDownloadCertificate cmd) { - String certificate = cmd.getCertificate(); - String certificateName = cmd.getCertificateName(); - s_logger.info("Importing certificate " + certificateName + " into keystore"); - - final File agentFile = PropertiesUtil.findConfigFile("agent.properties"); - if (agentFile == null) { - return new Answer(cmd, false, "Failed to find agent.properties file"); - } - - final String keyStoreFile = agentFile.getParent() + "/" + KeyStoreUtils.KS_FILENAME; - - String cerFile = agentFile.getParent() + "/" + certificateName + ".cer"; - Script.runSimpleBashScript(String.format("echo '%s' > %s", certificate, cerFile)); - - String privatePasswordFormat = "sed -n '/keystore.passphrase/p' '%s' 2>/dev/null | sed 's/keystore.passphrase=//g' 2>/dev/null"; - String privatePasswordCmd = String.format(privatePasswordFormat, agentFile.getAbsolutePath()); - String privatePassword = Script.runSimpleBashScript(privatePasswordCmd); - - String importCommandFormat = "keytool -importcert -file %s -keystore %s -alias '%s' -storepass '%s' -noprompt"; - String importCmd = String.format(importCommandFormat, cerFile, keyStoreFile, certificateName, privatePassword); - Script.runSimpleBashScript(importCmd); - return new Answer(cmd, true, "Certificate " + certificateName + " imported"); - } - public Answer setupAgentKeystore(final SetupKeyStoreCommand cmd) { final String keyStorePassword = cmd.getKeystorePassword(); final long validityDays = cmd.getValidityDays(); diff --git a/api/src/main/java/com/cloud/network/element/DhcpServiceProvider.java b/api/src/main/java/com/cloud/network/element/DhcpServiceProvider.java index 12830f8cec0..3f530c23d34 100644 --- a/api/src/main/java/com/cloud/network/element/DhcpServiceProvider.java +++ b/api/src/main/java/com/cloud/network/element/DhcpServiceProvider.java @@ -37,4 +37,6 @@ public interface DhcpServiceProvider extends NetworkElement { boolean removeDhcpSupportForSubnet(Network network) throws ResourceUnavailableException; boolean setExtraDhcpOptions(Network network, long nicId, Map dhcpOptions); + + boolean removeDhcpEntry(Network network, NicProfile nic, VirtualMachineProfile vmProfile) throws ResourceUnavailableException; } diff --git a/api/src/main/java/com/cloud/storage/MigrationOptions.java b/api/src/main/java/com/cloud/storage/MigrationOptions.java new file mode 100644 index 00000000000..38c1ee87bbe --- /dev/null +++ b/api/src/main/java/com/cloud/storage/MigrationOptions.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.cloud.storage; + +import java.io.Serializable; + +public class MigrationOptions implements Serializable { + + private String srcPoolUuid; + private Storage.StoragePoolType srcPoolType; + private Type type; + private String srcBackingFilePath; + private boolean copySrcTemplate; + private String srcVolumeUuid; + private int timeout; + + public enum Type { + LinkedClone, FullClone + } + + public MigrationOptions() { + } + + public MigrationOptions(String srcPoolUuid, Storage.StoragePoolType srcPoolType, String srcBackingFilePath, boolean copySrcTemplate) { + this.srcPoolUuid = srcPoolUuid; + this.srcPoolType = srcPoolType; + this.type = Type.LinkedClone; + this.srcBackingFilePath = srcBackingFilePath; + this.copySrcTemplate = copySrcTemplate; + } + + public MigrationOptions(String srcPoolUuid, Storage.StoragePoolType srcPoolType, String srcVolumeUuid) { + this.srcPoolUuid = srcPoolUuid; + this.srcPoolType = srcPoolType; + this.type = Type.FullClone; + this.srcVolumeUuid = srcVolumeUuid; + } + + public String getSrcPoolUuid() { + return srcPoolUuid; + } + + public Storage.StoragePoolType getSrcPoolType() { + return srcPoolType; + } + + public String getSrcBackingFilePath() { + return srcBackingFilePath; + } + + public boolean isCopySrcTemplate() { + return copySrcTemplate; + } + + public String getSrcVolumeUuid() { + return srcVolumeUuid; + } + + public Type getType() { + return type; + } + + public int getTimeout() { + return timeout; + } + + public void setTimeout(int timeout) { + this.timeout = timeout; + } +} diff --git a/api/src/main/java/com/cloud/template/TemplateApiService.java b/api/src/main/java/com/cloud/template/TemplateApiService.java index 7348547cee0..b62628560ae 100644 --- a/api/src/main/java/com/cloud/template/TemplateApiService.java +++ b/api/src/main/java/com/cloud/template/TemplateApiService.java @@ -24,6 +24,7 @@ import org.apache.cloudstack.api.BaseListTemplateOrIsoPermissionsCmd; import org.apache.cloudstack.api.BaseUpdateTemplateOrIsoPermissionsCmd; import org.apache.cloudstack.api.command.user.iso.DeleteIsoCmd; import org.apache.cloudstack.api.command.user.iso.ExtractIsoCmd; +import org.apache.cloudstack.api.command.user.iso.GetUploadParamsForIsoCmd; import org.apache.cloudstack.api.command.user.iso.RegisterIsoCmd; import org.apache.cloudstack.api.command.user.iso.UpdateIsoCmd; import org.apache.cloudstack.api.command.user.template.CopyTemplateCmd; @@ -45,10 +46,12 @@ public interface TemplateApiService { VirtualMachineTemplate registerTemplate(RegisterTemplateCmd cmd) throws URISyntaxException, ResourceAllocationException; - public GetUploadParamsResponse registerTemplateForPostUpload(GetUploadParamsForTemplateCmd cmd) throws ResourceAllocationException, MalformedURLException; + GetUploadParamsResponse registerTemplateForPostUpload(GetUploadParamsForTemplateCmd cmd) throws ResourceAllocationException, MalformedURLException; VirtualMachineTemplate registerIso(RegisterIsoCmd cmd) throws IllegalArgumentException, ResourceAllocationException; + GetUploadParamsResponse registerIsoForPostUpload(GetUploadParamsForIsoCmd cmd) throws ResourceAllocationException, MalformedURLException; + VirtualMachineTemplate copyTemplate(CopyTemplateCmd cmd) throws StorageUnavailableException, ResourceAllocationException; VirtualMachineTemplate prepareTemplate(long templateId, long zoneId, Long storageId); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/direct/download/UploadTemplateDirectDownloadCertificate.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/direct/download/UploadTemplateDirectDownloadCertificateCmd.java old mode 100644 new mode 100755 similarity index 81% rename from api/src/main/java/org/apache/cloudstack/api/command/admin/direct/download/UploadTemplateDirectDownloadCertificate.java rename to api/src/main/java/org/apache/cloudstack/api/command/admin/direct/download/UploadTemplateDirectDownloadCertificateCmd.java index 60d4262546c..c93fca2d300 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/direct/download/UploadTemplateDirectDownloadCertificate.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/direct/download/UploadTemplateDirectDownloadCertificateCmd.java @@ -1,90 +1,87 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -package org.apache.cloudstack.api.command.admin.direct.download; - -import com.cloud.exception.ConcurrentOperationException; -import com.cloud.exception.InsufficientCapacityException; -import com.cloud.exception.ResourceAllocationException; -import com.cloud.exception.ResourceUnavailableException; -import com.cloud.exception.NetworkRuleConflictException; -import org.apache.cloudstack.acl.RoleType; -import org.apache.cloudstack.api.APICommand; -import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.BaseCmd; -import org.apache.cloudstack.api.Parameter; -import org.apache.cloudstack.api.ServerApiException; -import org.apache.cloudstack.api.ApiErrorCode; -import org.apache.cloudstack.api.response.SuccessResponse; -import org.apache.cloudstack.context.CallContext; -import org.apache.cloudstack.direct.download.DirectDownloadManager; -import org.apache.log4j.Logger; - -import javax.inject.Inject; - -@APICommand(name = UploadTemplateDirectDownloadCertificate.APINAME, - description = "Upload a certificate for HTTPS direct template download on KVM hosts", - responseObject = SuccessResponse.class, - requestHasSensitiveInfo = true, - responseHasSensitiveInfo = true, - since = "4.11.0", - authorized = {RoleType.Admin}) -public class UploadTemplateDirectDownloadCertificate extends BaseCmd { - - @Inject - DirectDownloadManager directDownloadManager; - - private static final Logger LOG = Logger.getLogger(UploadTemplateDirectDownloadCertificate.class); - public static final String APINAME = "uploadTemplateDirectDownloadCertificate"; - - @Parameter(name = ApiConstants.CERTIFICATE, type = BaseCmd.CommandType.STRING, required = true, length = 65535, - description = "SSL certificate") - private String certificate; - - @Parameter(name = ApiConstants.NAME , type = BaseCmd.CommandType.STRING, required = true, - description = "Name for the uploaded certificate") - private String name; - - @Parameter(name = ApiConstants.HYPERVISOR, type = BaseCmd.CommandType.STRING, required = true, description = "Hypervisor type") - private String hypervisor; - - @Override - public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { - if (!hypervisor.equalsIgnoreCase("kvm")) { - throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Currently supporting KVM hosts only"); - } - - SuccessResponse response = new SuccessResponse(getCommandName()); - try { - LOG.debug("Uploading certificate " + name + " to agents for Direct Download"); - boolean result = directDownloadManager.uploadCertificateToHosts(certificate, name, hypervisor); - response.setSuccess(result); - setResponseObject(response); - } catch (Exception e) { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage()); - } - } - - @Override - public String getCommandName() { - return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX; - } - - @Override - public long getEntityOwnerId() { - return CallContext.current().getCallingAccount().getId(); - } -} +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.direct.download; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.direct.download.DirectDownloadManager; +import org.apache.log4j.Logger; + +import javax.inject.Inject; + +@APICommand(name = UploadTemplateDirectDownloadCertificateCmd.APINAME, + description = "Upload a certificate for HTTPS direct template download on KVM hosts", + responseObject = SuccessResponse.class, + requestHasSensitiveInfo = true, + responseHasSensitiveInfo = true, + since = "4.11.0", + authorized = {RoleType.Admin}) +public class UploadTemplateDirectDownloadCertificateCmd extends BaseCmd { + + @Inject + DirectDownloadManager directDownloadManager; + + private static final Logger LOG = Logger.getLogger(UploadTemplateDirectDownloadCertificateCmd.class); + public static final String APINAME = "uploadTemplateDirectDownloadCertificate"; + + @Parameter(name = ApiConstants.CERTIFICATE, type = BaseCmd.CommandType.STRING, required = true, length = 65535, + description = "SSL certificate") + private String certificate; + + @Parameter(name = ApiConstants.NAME , type = BaseCmd.CommandType.STRING, required = true, + description = "Name for the uploaded certificate") + private String name; + + @Parameter(name = ApiConstants.HYPERVISOR, type = BaseCmd.CommandType.STRING, required = true, description = "Hypervisor type") + private String hypervisor; + + @Override + public void execute() { + if (!hypervisor.equalsIgnoreCase("kvm")) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Currently supporting KVM hosts only"); + } + + try { + LOG.debug("Uploading certificate " + name + " to agents for Direct Download"); + boolean result = directDownloadManager.uploadCertificateToHosts(certificate, name, hypervisor); + SuccessResponse response = new SuccessResponse(getCommandName()); + response.setSuccess(result); + setResponseObject(response); + } catch (Exception e) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage()); + } + } + + @Override + public String getCommandName() { + return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX; + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } +} + + diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/address/ListPublicIpAddressesCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/address/ListPublicIpAddressesCmd.java index d590081104a..d25d167636f 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/address/ListPublicIpAddressesCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/address/ListPublicIpAddressesCmd.java @@ -86,6 +86,13 @@ public class ListPublicIpAddressesCmd extends BaseListTaggedResourcesCmd { description = "lists all public IP addresses associated to the network specified") private Long associatedNetworkId; + @Parameter(name = ApiConstants.NETWORK_ID, + type = CommandType.UUID, + entityType = NetworkResponse.class, + description = "lists all public IP addresses by source network ID", + since = "4.13.0") + private Long networkId; + @Parameter(name = ApiConstants.IS_SOURCE_NAT, type = CommandType.BOOLEAN, description = "list only source NAT IP addresses") private Boolean isSourceNat; @@ -133,6 +140,10 @@ public class ListPublicIpAddressesCmd extends BaseListTaggedResourcesCmd { return associatedNetworkId; } + public Long getNetworkId() { + return networkId; + } + public Boolean isSourceNat() { return isSourceNat; } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/iso/GetUploadParamsForIsoCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/iso/GetUploadParamsForIsoCmd.java new file mode 100644 index 00000000000..92e3b979885 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/iso/GetUploadParamsForIsoCmd.java @@ -0,0 +1,158 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.iso; + +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.AbstractGetUploadParamsCmd; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.GetUploadParamsResponse; +import org.apache.cloudstack.api.response.GuestOSResponse; +import org.apache.cloudstack.api.response.ZoneResponse; +import org.apache.cloudstack.context.CallContext; + +import java.net.MalformedURLException; + +@APICommand(name = GetUploadParamsForIsoCmd.APINAME, + description = "upload an existing ISO into the CloudStack cloud.", + responseObject = GetUploadParamsResponse.class, since = "4.13", + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) +public class GetUploadParamsForIsoCmd extends AbstractGetUploadParamsCmd { + + public static final String APINAME = "getUploadParamsForIso"; + + private static final String s_name = "postuploadisoresponse"; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.BOOTABLE, type = BaseCmd.CommandType.BOOLEAN, description = "true if this ISO is bootable. If not passed explicitly its assumed to be true") + private Boolean bootable; + + @Parameter(name = ApiConstants.DISPLAY_TEXT, + type = BaseCmd.CommandType.STRING, + required = true, + description = "the display text of the ISO. This is usually used for display purposes.", + length = 4096) + private String displayText; + + @Parameter(name = ApiConstants.IS_FEATURED, type = BaseCmd.CommandType.BOOLEAN, description = "true if you want this ISO to be featured") + private Boolean featured; + + @Parameter(name = ApiConstants.IS_PUBLIC, + type = BaseCmd.CommandType.BOOLEAN, + description = "true if you want to register the ISO to be publicly available to all users, false otherwise.") + private Boolean publicIso; + + @Parameter(name = ApiConstants.IS_EXTRACTABLE, type = BaseCmd.CommandType.BOOLEAN, description = "true if the ISO or its derivatives are extractable; default is false") + private Boolean extractable; + + @Parameter(name = ApiConstants.NAME, type = BaseCmd.CommandType.STRING, required = true, description = "the name of the ISO") + private String isoName; + + @Parameter(name = ApiConstants.OS_TYPE_ID, + type = BaseCmd.CommandType.UUID, + entityType = GuestOSResponse.class, + description = "the ID of the OS type that best represents the OS of this ISO. If the ISO is bootable this parameter needs to be passed") + private Long osTypeId; + + @Parameter(name=ApiConstants.ZONE_ID, type= BaseCmd.CommandType.UUID, entityType = ZoneResponse.class, + required=true, description="the ID of the zone you wish to register the ISO to.") + protected Long zoneId; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Boolean isBootable() { + return bootable; + } + + public String getDisplayText() { + return displayText; + } + + public Boolean isFeatured() { + return featured; + } + + public Boolean isPublic() { + return publicIso; + } + + public Boolean isExtractable() { + return extractable; + } + + public String getIsoName() { + return isoName; + } + + public Long getOsTypeId() { + return osTypeId; + } + + public Long getZoneId() { + return zoneId; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { + validateRequest(); + try { + GetUploadParamsResponse response = _templateService.registerIsoForPostUpload(this); + response.setResponseName(getCommandName()); + setResponseObject(response); + } catch (ResourceAllocationException | MalformedURLException e) { + s_logger.error("Exception while registering template", e); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Exception while registering ISO: " + e.getMessage()); + } + } + + private void validateRequest() { + if (getZoneId() <= 0) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Invalid zoneid"); + } + } + + @Override + public String getCommandName() { + return s_name; + } + + @Override + public long getEntityOwnerId() { + Long accountId = _accountService.finalyzeAccountId(getAccountName(), getDomainId(), getProjectId(), true); + if (accountId == null) { + return CallContext.current().getCallingAccount().getId(); + } + return accountId; + } +} \ No newline at end of file diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java index 8c2a7e41754..b77b9854590 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java @@ -16,15 +16,6 @@ // under the License. package org.apache.cloudstack.api.command.user.snapshot; -import com.cloud.event.EventTypes; -import com.cloud.exception.InvalidParameterValueException; -import com.cloud.exception.PermissionDeniedException; -import com.cloud.exception.ResourceAllocationException; -import com.cloud.projects.Project; -import com.cloud.storage.Snapshot; -import com.cloud.storage.Volume; -import com.cloud.user.Account; -import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiCommandJobType; import org.apache.cloudstack.api.ApiConstants; @@ -39,6 +30,16 @@ import org.apache.cloudstack.api.response.SnapshotResponse; import org.apache.cloudstack.api.response.VolumeResponse; import org.apache.log4j.Logger; +import com.cloud.event.EventTypes; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.PermissionDeniedException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.projects.Project; +import com.cloud.storage.Snapshot; +import com.cloud.storage.Volume; +import com.cloud.user.Account; +import com.cloud.utils.exception.CloudRuntimeException; + @APICommand(name = "createSnapshot", description = "Creates an instant snapshot of a volume.", responseObject = SnapshotResponse.class, entityType = {Snapshot.class}, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) public class CreateSnapshotCmd extends BaseAsyncCreateCmd { @@ -171,7 +172,7 @@ public class CreateSnapshotCmd extends BaseAsyncCreateCmd { @Override public String getEventDescription() { - return "creating snapshot for volume: " + this._uuidMgr.getUuid(Volume.class, getVolumeId()); + return "creating snapshot for volume: " + getVolumeUuid(); } @Override @@ -186,7 +187,7 @@ public class CreateSnapshotCmd extends BaseAsyncCreateCmd { setEntityId(snapshot.getId()); setEntityUuid(snapshot.getUuid()); } else { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to create snapshot"); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to create snapshot for volume" + getVolumeUuid()); } } @@ -202,10 +203,10 @@ public class CreateSnapshotCmd extends BaseAsyncCreateCmd { response.setResponseName(getCommandName()); setResponseObject(response); } else { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to create snapshot due to an internal error creating snapshot for volume " + getVolumeId()); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to create snapshot due to an internal error creating snapshot for volume " + getVolumeUuid()); } } catch (Exception e) { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to create snapshot due to an internal error creating snapshot for volume " + getVolumeId()); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to create snapshot due to an internal error creating snapshot for volume " + getVolumeUuid()); } } @@ -249,4 +250,8 @@ public class CreateSnapshotCmd extends BaseAsyncCreateCmd { return asyncBackup; } } + + protected String getVolumeUuid() { + return _uuidMgr.getUuid(Volume.class, getVolumeId()); + } } diff --git a/api/src/test/java/org/apache/cloudstack/api/command/test/CreateSnapshotCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/test/CreateSnapshotCmdTest.java index bc2beb6cb66..ceb63ab6e56 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/test/CreateSnapshotCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/test/CreateSnapshotCmdTest.java @@ -16,11 +16,12 @@ // under the License. package org.apache.cloudstack.api.command.test; -import com.cloud.storage.Snapshot; -import com.cloud.storage.VolumeApiService; -import com.cloud.user.Account; -import com.cloud.user.AccountService; -import junit.framework.TestCase; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyBoolean; +import static org.mockito.Matchers.anyLong; +import static org.mockito.Matchers.anyString; +import static org.mockito.Matchers.isNull; + import org.apache.cloudstack.api.ResponseGenerator; import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.command.user.snapshot.CreateSnapshotCmd; @@ -32,11 +33,12 @@ import org.junit.Test; import org.junit.rules.ExpectedException; import org.mockito.Mockito; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyBoolean; -import static org.mockito.Matchers.anyLong; -import static org.mockito.Matchers.anyString; -import static org.mockito.Matchers.isNull; +import com.cloud.storage.Snapshot; +import com.cloud.storage.VolumeApiService; +import com.cloud.user.Account; +import com.cloud.user.AccountService; + +import junit.framework.TestCase; public class CreateSnapshotCmdTest extends TestCase { @@ -66,6 +68,11 @@ public class CreateSnapshotCmdTest extends TestCase { public long getEntityOwnerId(){ return 1L; } + + @Override + protected String getVolumeUuid() { + return "123"; + } }; } @@ -126,7 +133,7 @@ public class CreateSnapshotCmdTest extends TestCase { try { createSnapshotCmd.execute(); } catch (ServerApiException exception) { - Assert.assertEquals("Failed to create snapshot due to an internal error creating snapshot for volume 1", exception.getDescription()); + Assert.assertEquals("Failed to create snapshot due to an internal error creating snapshot for volume 123", exception.getDescription()); } } } diff --git a/client/conf/ehcache.xml.in b/client/conf/ehcache.xml.in index 19bfd0f6967..4f25978ed00 100755 --- a/client/conf/ehcache.xml.in +++ b/client/conf/ehcache.xml.in @@ -163,14 +163,6 @@ under the License. used. --> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + diff --git a/engine/schema/src/test/java/com/cloud/upgrade/DatabaseUpgradeCheckerTest.java b/engine/schema/src/test/java/com/cloud/upgrade/DatabaseUpgradeCheckerTest.java index 65e222df10a..982a386161b 100644 --- a/engine/schema/src/test/java/com/cloud/upgrade/DatabaseUpgradeCheckerTest.java +++ b/engine/schema/src/test/java/com/cloud/upgrade/DatabaseUpgradeCheckerTest.java @@ -30,6 +30,7 @@ import com.cloud.upgrade.dao.DbUpgrade; import com.cloud.upgrade.dao.Upgrade41000to41100; import com.cloud.upgrade.dao.Upgrade41100to41110; import com.cloud.upgrade.dao.Upgrade41110to41120; +import com.cloud.upgrade.dao.Upgrade41120to41130; import com.cloud.upgrade.dao.Upgrade41120to41200; import com.cloud.upgrade.dao.Upgrade452to453; import com.cloud.upgrade.dao.Upgrade453to460; @@ -98,10 +99,11 @@ public class DatabaseUpgradeCheckerTest { assertTrue(upgrades[0] instanceof Upgrade41000to41100); assertTrue(upgrades[1] instanceof Upgrade41100to41110); assertTrue(upgrades[2] instanceof Upgrade41110to41120); - assertTrue(upgrades[3] instanceof Upgrade41120to41200); + assertTrue(upgrades[3] instanceof Upgrade41120to41130); + assertTrue(upgrades[4] instanceof Upgrade41120to41200); assertTrue(Arrays.equals(new String[] {"4.11.0.0", "4.11.1.0"}, upgrades[1].getUpgradableVersionRange())); - assertEquals(currentVersion.toString(), upgrades[3].getUpgradedVersion()); + assertEquals(currentVersion.toString(), upgrades[4].getUpgradedVersion()); } diff --git a/engine/storage/configdrive/test/org/apache/cloudstack/storage/configdrive/ConfigDriveTest.java b/engine/storage/configdrive/src/test/java/org/apache/cloudstack/storage/configdrive/ConfigDriveTest.java similarity index 100% rename from engine/storage/configdrive/test/org/apache/cloudstack/storage/configdrive/ConfigDriveTest.java rename to engine/storage/configdrive/src/test/java/org/apache/cloudstack/storage/configdrive/ConfigDriveTest.java diff --git a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageDataMotionStrategy.java b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageDataMotionStrategy.java index 1bbe1770809..e42715a1e6d 100644 --- a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageDataMotionStrategy.java +++ b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageDataMotionStrategy.java @@ -24,6 +24,8 @@ import java.util.Set; import javax.inject.Inject; +import com.cloud.storage.ScopeType; +import com.cloud.storage.Storage; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.StrategyPriority; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory; @@ -54,6 +56,7 @@ import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.VMTemplatePoolDao; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VirtualMachineManager; +import org.apache.commons.collections.MapUtils; /** * Extends {@link StorageSystemDataMotionStrategy}, allowing KVM hosts to migrate VMs with the ROOT volume on a non managed local storage pool. @@ -77,14 +80,16 @@ public class KvmNonManagedStorageDataMotionStrategy extends StorageSystemDataMot * Note that the super implementation (override) is called by {@link #canHandle(Map, Host, Host)} which ensures that {@link #internalCanHandle(Map)} will be executed only if the source host is KVM. */ @Override - protected StrategyPriority internalCanHandle(Map volumeMap) { - if (super.internalCanHandle(volumeMap) == StrategyPriority.CANT_HANDLE) { - Set volumeInfoSet = volumeMap.keySet(); + protected StrategyPriority internalCanHandle(Map volumeMap, Host srcHost, Host destHost) { + if (super.internalCanHandle(volumeMap, srcHost, destHost) == StrategyPriority.CANT_HANDLE) { + if (canHandleKVMNonManagedLiveNFSStorageMigration(volumeMap, srcHost, destHost) == StrategyPriority.CANT_HANDLE) { + Set volumeInfoSet = volumeMap.keySet(); - for (VolumeInfo volumeInfo : volumeInfoSet) { - StoragePoolVO storagePoolVO = _storagePoolDao.findById(volumeInfo.getPoolId()); - if (storagePoolVO.getPoolType() != StoragePoolType.Filesystem && storagePoolVO.getPoolType() != StoragePoolType.NetworkFilesystem) { - return StrategyPriority.CANT_HANDLE; + for (VolumeInfo volumeInfo : volumeInfoSet) { + StoragePoolVO storagePoolVO = _storagePoolDao.findById(volumeInfo.getPoolId()); + if (storagePoolVO.getPoolType() != StoragePoolType.Filesystem && storagePoolVO.getPoolType() != StoragePoolType.NetworkFilesystem) { + return StrategyPriority.CANT_HANDLE; + } } } return StrategyPriority.HYPERVISOR; @@ -92,6 +97,52 @@ public class KvmNonManagedStorageDataMotionStrategy extends StorageSystemDataMot return StrategyPriority.CANT_HANDLE; } + /** + * Allow KVM live storage migration for non managed storage when: + * - Source host and destination host are different, and are on the same cluster + * - Source and destination storage are NFS + * - Destination storage is cluster-wide + */ + protected StrategyPriority canHandleKVMNonManagedLiveNFSStorageMigration(Map volumeMap, + Host srcHost, Host destHost) { + if (srcHost.getId() != destHost.getId() && + srcHost.getClusterId().equals(destHost.getClusterId()) && + isSourceNfsPrimaryStorage(volumeMap) && + isDestinationNfsPrimaryStorageClusterWide(volumeMap)) { + return StrategyPriority.HYPERVISOR; + } + return StrategyPriority.CANT_HANDLE; + } + + /** + * True if volumes source storage are NFS + */ + protected boolean isSourceNfsPrimaryStorage(Map volumeMap) { + if (MapUtils.isNotEmpty(volumeMap)) { + for (VolumeInfo volumeInfo : volumeMap.keySet()) { + StoragePoolVO storagePoolVO = _storagePoolDao.findById(volumeInfo.getPoolId()); + return storagePoolVO != null && + storagePoolVO.getPoolType() == Storage.StoragePoolType.NetworkFilesystem; + } + } + return false; + } + + /** + * True if destination storage is cluster-wide NFS + */ + protected boolean isDestinationNfsPrimaryStorageClusterWide(Map volumeMap) { + if (MapUtils.isNotEmpty(volumeMap)) { + for (DataStore dataStore : volumeMap.values()) { + StoragePoolVO storagePoolVO = _storagePoolDao.findById(dataStore.getId()); + return storagePoolVO != null && + storagePoolVO.getPoolType() == Storage.StoragePoolType.NetworkFilesystem && + storagePoolVO.getScope() == ScopeType.CLUSTER; + } + } + return false; + } + /** * Configures a {@link MigrateDiskInfo} object configured for migrating a File System volume and calls rootImageProvisioning. */ @@ -135,7 +186,7 @@ public class KvmNonManagedStorageDataMotionStrategy extends StorageSystemDataMot */ @Override protected boolean shouldMigrateVolume(StoragePoolVO sourceStoragePool, Host destHost, StoragePoolVO destStoragePool) { - return sourceStoragePool.getPoolType() == StoragePoolType.Filesystem; + return sourceStoragePool.getPoolType() == StoragePoolType.Filesystem || sourceStoragePool.getPoolType() == StoragePoolType.NetworkFilesystem; } /** diff --git a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java index 1f3368fd1bb..45cd2954f15 100644 --- a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java +++ b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java @@ -18,62 +18,18 @@ */ package org.apache.cloudstack.storage.motion; -import com.cloud.agent.AgentManager; -import com.cloud.agent.api.Answer; -import com.cloud.agent.api.storage.CopyVolumeAnswer; -import com.cloud.agent.api.storage.CopyVolumeCommand; -import com.cloud.agent.api.MigrateAnswer; -import com.cloud.agent.api.MigrateCommand; -import com.cloud.agent.api.MigrateCommand.MigrateDiskInfo; -import com.cloud.agent.api.ModifyTargetsAnswer; -import com.cloud.agent.api.ModifyTargetsCommand; -import com.cloud.agent.api.PrepareForMigrationCommand; -import com.cloud.agent.api.storage.MigrateVolumeAnswer; -import com.cloud.agent.api.storage.MigrateVolumeCommand; -import com.cloud.agent.api.to.DataStoreTO; -import com.cloud.agent.api.to.DataTO; -import com.cloud.agent.api.to.DiskTO; -import com.cloud.agent.api.to.NfsTO; -import com.cloud.agent.api.to.VirtualMachineTO; -import com.cloud.configuration.Config; -import com.cloud.dc.dao.ClusterDao; -import com.cloud.exception.AgentUnavailableException; -import com.cloud.exception.OperationTimedoutException; -import com.cloud.host.Host; -import com.cloud.host.HostVO; -import com.cloud.host.dao.HostDao; -import com.cloud.hypervisor.Hypervisor.HypervisorType; -import com.cloud.resource.ResourceState; -import com.cloud.storage.DataStoreRole; -import com.cloud.storage.DiskOfferingVO; -import com.cloud.storage.Snapshot; -import com.cloud.storage.SnapshotVO; -import com.cloud.storage.Storage.ImageFormat; -import com.cloud.storage.Storage.StoragePoolType; -import com.cloud.storage.StorageManager; -import com.cloud.storage.StoragePool; -import com.cloud.storage.VMTemplateVO; -import com.cloud.storage.VolumeDetailVO; -import com.cloud.storage.Volume; -import com.cloud.storage.VolumeVO; -import com.cloud.storage.dao.DiskOfferingDao; -import com.cloud.storage.dao.GuestOSCategoryDao; -import com.cloud.storage.dao.GuestOSDao; -import com.cloud.storage.dao.SnapshotDao; -import com.cloud.storage.dao.SnapshotDetailsDao; -import com.cloud.storage.dao.SnapshotDetailsVO; -import com.cloud.storage.dao.VMTemplateDao; -import com.cloud.storage.dao.VolumeDao; -import com.cloud.storage.dao.VolumeDetailsDao; -import com.cloud.utils.NumbersUtil; -import com.cloud.utils.db.GlobalLock; -import com.cloud.utils.exception.CloudRuntimeException; -import com.cloud.vm.VirtualMachine; -import com.cloud.vm.VirtualMachineManager; -import com.cloud.vm.VMInstanceVO; -import com.cloud.vm.dao.VMInstanceDao; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.TimeUnit; -import com.google.common.base.Preconditions; +import javax.inject.Inject; import org.apache.cloudstack.engine.subsystem.api.storage.ChapInfo; import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope; @@ -86,6 +42,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; import org.apache.cloudstack.engine.subsystem.api.storage.HostScope; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.Event; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver; import org.apache.cloudstack.engine.subsystem.api.storage.Scope; @@ -111,25 +68,73 @@ import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; -import org.springframework.stereotype.Component; +import com.cloud.agent.AgentManager; +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.MigrateAnswer; +import com.cloud.agent.api.MigrateCommand; +import com.cloud.agent.api.MigrateCommand.MigrateDiskInfo; +import com.cloud.agent.api.ModifyTargetsAnswer; +import com.cloud.agent.api.ModifyTargetsCommand; +import com.cloud.agent.api.PrepareForMigrationCommand; +import com.cloud.agent.api.storage.CheckStorageAvailabilityCommand; +import com.cloud.agent.api.storage.CopyVolumeAnswer; +import com.cloud.agent.api.storage.CopyVolumeCommand; +import com.cloud.agent.api.storage.MigrateVolumeAnswer; +import com.cloud.agent.api.storage.MigrateVolumeCommand; +import com.cloud.agent.api.to.DataStoreTO; +import com.cloud.agent.api.to.DataTO; +import com.cloud.agent.api.to.DiskTO; +import com.cloud.agent.api.to.NfsTO; +import com.cloud.agent.api.to.VirtualMachineTO; +import com.cloud.configuration.Config; +import com.cloud.dc.dao.ClusterDao; +import com.cloud.exception.AgentUnavailableException; +import com.cloud.exception.OperationTimedoutException; +import com.cloud.host.Host; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDao; +import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.resource.ResourceState; +import com.cloud.storage.DataStoreRole; +import com.cloud.storage.DiskOfferingVO; +import com.cloud.storage.MigrationOptions; +import com.cloud.storage.ScopeType; +import com.cloud.storage.Snapshot; +import com.cloud.storage.SnapshotVO; +import com.cloud.storage.Storage; +import com.cloud.storage.Storage.ImageFormat; +import com.cloud.storage.Storage.StoragePoolType; +import com.cloud.storage.StorageManager; +import com.cloud.storage.StoragePool; +import com.cloud.storage.VMTemplateStoragePoolVO; +import com.cloud.storage.VMTemplateStorageResourceAssoc; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.Volume; +import com.cloud.storage.VolumeDetailVO; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.DiskOfferingDao; +import com.cloud.storage.dao.GuestOSCategoryDao; +import com.cloud.storage.dao.GuestOSDao; +import com.cloud.storage.dao.SnapshotDao; +import com.cloud.storage.dao.SnapshotDetailsDao; +import com.cloud.storage.dao.SnapshotDetailsVO; +import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.storage.dao.VMTemplatePoolDao; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.storage.dao.VolumeDetailsDao; +import com.cloud.utils.NumbersUtil; +import com.cloud.utils.db.GlobalLock; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.VirtualMachineManager; +import com.cloud.vm.dao.VMInstanceDao; +import com.google.common.base.Preconditions; -import javax.inject.Inject; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.TimeUnit; - -@Component public class StorageSystemDataMotionStrategy implements DataMotionStrategy { private static final Logger LOGGER = Logger.getLogger(StorageSystemDataMotionStrategy.class); private static final Random RANDOM = new Random(System.nanoTime()); @@ -176,6 +181,8 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { private StorageCacheManager cacheMgr; @Inject private EndPointSelector selector; + @Inject + VMTemplatePoolDao templatePoolDao; @Override public StrategyPriority canHandle(DataObject srcData, DataObject destData) { @@ -272,7 +279,7 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { @Override public final StrategyPriority canHandle(Map volumeMap, Host srcHost, Host destHost) { if (HypervisorType.KVM.equals(srcHost.getHypervisorType())) { - return internalCanHandle(volumeMap); + return internalCanHandle(volumeMap, srcHost, destHost); } return StrategyPriority.CANT_HANDLE; } @@ -280,7 +287,7 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { /** * Handles migrating volumes on managed Storage. */ - protected StrategyPriority internalCanHandle(Map volumeMap) { + protected StrategyPriority internalCanHandle(Map volumeMap, Host srcHost, Host destHost) { Set volumeInfoSet = volumeMap.keySet(); for (VolumeInfo volumeInfo : volumeInfoSet) { @@ -299,6 +306,7 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { if (storagePoolVO.isManaged()) { return StrategyPriority.HIGHEST; } + } return StrategyPriority.CANT_HANDLE; } @@ -1698,6 +1706,50 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { return _snapshotDetailsDao.persist(snapshotDetails); } + /** + * Return expected MigrationOptions for a linked clone volume live storage migration + */ + protected MigrationOptions createLinkedCloneMigrationOptions(VolumeInfo srcVolumeInfo, VolumeInfo destVolumeInfo, String srcVolumeBackingFile, String srcPoolUuid, Storage.StoragePoolType srcPoolType) { + VMTemplateStoragePoolVO ref = templatePoolDao.findByPoolTemplate(destVolumeInfo.getPoolId(), srcVolumeInfo.getTemplateId()); + boolean updateBackingFileReference = ref == null; + String backingFile = ref != null ? ref.getInstallPath() : srcVolumeBackingFile; + return new MigrationOptions(srcPoolUuid, srcPoolType, backingFile, updateBackingFileReference); + } + + /** + * Return expected MigrationOptions for a full clone volume live storage migration + */ + protected MigrationOptions createFullCloneMigrationOptions(VolumeInfo srcVolumeInfo, VirtualMachineTO vmTO, Host srcHost, String srcPoolUuid, Storage.StoragePoolType srcPoolType) { + return new MigrationOptions(srcPoolUuid, srcPoolType, srcVolumeInfo.getPath()); + } + + /** + * Prepare hosts for KVM live storage migration depending on volume type by setting MigrationOptions on destination volume: + * - Linked clones (backing file on disk): Decide if template (backing file) should be copied to destination storage prior disk creation + * - Full clones (no backing file): Take snapshot of the VM prior disk creation + * Return this information + */ + protected void setVolumeMigrationOptions(VolumeInfo srcVolumeInfo, VolumeInfo destVolumeInfo, + VirtualMachineTO vmTO, Host srcHost, StoragePoolVO destStoragePool) { + if (!destStoragePool.isManaged()) { + String srcVolumeBackingFile = getVolumeBackingFile(srcVolumeInfo); + + String srcPoolUuid = srcVolumeInfo.getDataStore().getUuid(); + StoragePoolVO srcPool = _storagePoolDao.findById(srcVolumeInfo.getPoolId()); + Storage.StoragePoolType srcPoolType = srcPool.getPoolType(); + + MigrationOptions migrationOptions; + if (StringUtils.isNotBlank(srcVolumeBackingFile)) { + migrationOptions = createLinkedCloneMigrationOptions(srcVolumeInfo, destVolumeInfo, + srcVolumeBackingFile, srcPoolUuid, srcPoolType); + } else { + migrationOptions = createFullCloneMigrationOptions(srcVolumeInfo, vmTO, srcHost, srcPoolUuid, srcPoolType); + } + migrationOptions.setTimeout(StorageManager.KvmStorageOnlineMigrationWait.value()); + destVolumeInfo.setMigrationOptions(migrationOptions); + } + } + /** * For each disk to migrate: *
    @@ -1716,7 +1768,7 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { throw new CloudRuntimeException("Invalid hypervisor type (only KVM supported for this operation at the time being)"); } - verifyLiveMigrationMapForKVM(volumeDataStoreMap); + verifyLiveMigrationForKVM(volumeDataStoreMap, destHost); VMInstanceVO vmInstance = _vmDao.findById(vmTO.getId()); vmTO.setState(vmInstance.getState()); @@ -1725,6 +1777,7 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { Map migrateStorage = new HashMap<>(); Map srcVolumeInfoToDestVolumeInfo = new HashMap<>(); + boolean managedStorageDestination = false; for (Map.Entry entry : volumeDataStoreMap.entrySet()) { VolumeInfo srcVolumeInfo = entry.getKey(); DataStore destDataStore = entry.getValue(); @@ -1749,15 +1802,23 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { // move the volume from Ready to Migrating destVolumeInfo.processEvent(Event.MigrationRequested); + setVolumeMigrationOptions(srcVolumeInfo, destVolumeInfo, vmTO, srcHost, destStoragePool); + // create a volume on the destination storage destDataStore.getDriver().createAsync(destDataStore, destVolumeInfo, null); + managedStorageDestination = destStoragePool.isManaged(); + String volumeIdentifier = managedStorageDestination ? destVolumeInfo.get_iScsiName() : destVolumeInfo.getUuid(); + destVolume = _volumeDao.findById(destVolume.getId()); + destVolume.setPath(volumeIdentifier); setVolumePath(destVolume); _volumeDao.update(destVolume.getId(), destVolume); + postVolumeCreationActions(srcVolumeInfo, destVolumeInfo, vmTO, srcHost); + destVolumeInfo = _volumeDataFactory.getVolume(destVolume.getId(), destDataStore); handleQualityOfServiceForVolumeMigration(destVolumeInfo, PrimaryDataStoreDriver.QualityOfServiceState.MIGRATION); @@ -1766,9 +1827,18 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { String destPath = generateDestPath(destHost, destStoragePool, destVolumeInfo); - MigrateCommand.MigrateDiskInfo migrateDiskInfo = configureMigrateDiskInfo(srcVolumeInfo, destPath); - migrateDiskInfo.setSourceDiskOnStorageFileSystem(isStoragePoolTypeOfFile(sourceStoragePool)); - migrateDiskInfoList.add(migrateDiskInfo); + MigrateCommand.MigrateDiskInfo migrateDiskInfo; + if (managedStorageDestination) { + migrateDiskInfo = configureMigrateDiskInfo(srcVolumeInfo, destPath); + migrateDiskInfo.setSourceDiskOnStorageFileSystem(isStoragePoolTypeOfFile(sourceStoragePool)); + migrateDiskInfoList.add(migrateDiskInfo); + } else { + migrateDiskInfo = new MigrateCommand.MigrateDiskInfo(srcVolumeInfo.getPath(), + MigrateCommand.MigrateDiskInfo.DiskType.FILE, + MigrateCommand.MigrateDiskInfo.DriverType.QCOW2, + MigrateCommand.MigrateDiskInfo.Source.FILE, + connectHostToVolume(destHost, destVolumeInfo.getPoolId(), volumeIdentifier)); + } migrateStorage.put(srcVolumeInfo.getPath(), migrateDiskInfo); @@ -1795,15 +1865,12 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { boolean isWindows = _guestOsCategoryDao.findById(_guestOsDao.findById(vm.getGuestOSId()).getCategoryId()).getName().equalsIgnoreCase("Windows"); MigrateCommand migrateCommand = new MigrateCommand(vmTO.getName(), destHost.getPrivateIpAddress(), isWindows, vmTO, true); - migrateCommand.setWait(StorageManager.KvmStorageOnlineMigrationWait.value()); - migrateCommand.setMigrateStorage(migrateStorage); migrateCommand.setMigrateDiskInfoList(migrateDiskInfoList); + migrateCommand.setMigrateStorageManaged(managedStorageDestination); - String autoConvergence = _configDao.getValue(Config.KvmAutoConvergence.toString()); - boolean kvmAutoConvergence = Boolean.parseBoolean(autoConvergence); - + boolean kvmAutoConvergence = StorageManager.KvmAutoConvergence.value(); migrateCommand.setAutoConvergence(kvmAutoConvergence); MigrateAnswer migrateAnswer = (MigrateAnswer)agentManager.send(srcHost.getId(), migrateCommand); @@ -1863,7 +1930,9 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { * Configures a {@link MigrateDiskInfo} object with disk type of BLOCK, Driver type RAW and Source DEV */ protected MigrateCommand.MigrateDiskInfo configureMigrateDiskInfo(VolumeInfo srcVolumeInfo, String destPath) { - return new MigrateCommand.MigrateDiskInfo(srcVolumeInfo.getPath(), MigrateCommand.MigrateDiskInfo.DiskType.BLOCK, MigrateCommand.MigrateDiskInfo.DriverType.RAW, + return new MigrateCommand.MigrateDiskInfo(srcVolumeInfo.getPath(), + MigrateCommand.MigrateDiskInfo.DiskType.BLOCK, + MigrateCommand.MigrateDiskInfo.DriverType.RAW, MigrateCommand.MigrateDiskInfo.Source.DEV, destPath); } @@ -1883,6 +1952,21 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { // This method is used by classes that extend this one } + /* + * Return backing file for volume (if any), only for KVM volumes + */ + private String getVolumeBackingFile(VolumeInfo srcVolumeInfo) { + if (srcVolumeInfo.getHypervisorType() == HypervisorType.KVM && + srcVolumeInfo.getTemplateId() != null && srcVolumeInfo.getPoolId() != null) { + VMTemplateVO template = _vmTemplateDao.findById(srcVolumeInfo.getTemplateId()); + if (template.getFormat() != null && template.getFormat() != Storage.ImageFormat.ISO) { + VMTemplateStoragePoolVO ref = templatePoolDao.findByPoolTemplate(srcVolumeInfo.getPoolId(), srcVolumeInfo.getTemplateId()); + return ref != null ? ref.getInstallPath() : null; + } + } + return null; + } + private void handlePostMigration(boolean success, Map srcVolumeInfoToDestVolumeInfo, VirtualMachineTO vmTO, Host destHost) { if (!success) { try { @@ -2046,10 +2130,40 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { return modifyTargetsAnswer.getConnectedPaths(); } + /** + * Update reference on template_spool_ref table of copied template to destination storage + */ + protected void updateCopiedTemplateReference(VolumeInfo srcVolumeInfo, VolumeInfo destVolumeInfo) { + VMTemplateStoragePoolVO ref = templatePoolDao.findByPoolTemplate(srcVolumeInfo.getPoolId(), srcVolumeInfo.getTemplateId()); + VMTemplateStoragePoolVO newRef = new VMTemplateStoragePoolVO(destVolumeInfo.getPoolId(), ref.getTemplateId()); + newRef.setDownloadPercent(100); + newRef.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOADED); + newRef.setState(ObjectInDataStoreStateMachine.State.Ready); + newRef.setTemplateSize(ref.getTemplateSize()); + newRef.setLocalDownloadPath(ref.getLocalDownloadPath()); + newRef.setInstallPath(ref.getInstallPath()); + templatePoolDao.persist(newRef); + } + + /** + * Handle post destination volume creation actions depending on the migrating volume type: full clone or linked clone + */ + protected void postVolumeCreationActions(VolumeInfo srcVolumeInfo, VolumeInfo destVolumeInfo, VirtualMachineTO vmTO, Host srcHost) { + MigrationOptions migrationOptions = destVolumeInfo.getMigrationOptions(); + if (migrationOptions != null) { + if (migrationOptions.getType() == MigrationOptions.Type.LinkedClone && migrationOptions.isCopySrcTemplate()) { + updateCopiedTemplateReference(srcVolumeInfo, destVolumeInfo); + } + } + } + /* - * At a high level: The source storage cannot be managed and the destination storage must be managed. + * At a high level: The source storage cannot be managed and + * the destination storages can be all managed or all not managed, not mixed. */ - private void verifyLiveMigrationMapForKVM(Map volumeDataStoreMap) { + protected void verifyLiveMigrationForKVM(Map volumeDataStoreMap, Host destHost) { + Boolean storageTypeConsistency = null; + Map sourcePools = new HashMap<>(); for (Map.Entry entry : volumeDataStoreMap.entrySet()) { VolumeInfo volumeInfo = entry.getKey(); @@ -2070,6 +2184,47 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { if (destStoragePoolVO == null) { throw new CloudRuntimeException("Destination storage pool with ID " + dataStore.getId() + " was not located."); } + + if (storageTypeConsistency == null) { + storageTypeConsistency = destStoragePoolVO.isManaged(); + } else if (storageTypeConsistency != destStoragePoolVO.isManaged()) { + throw new CloudRuntimeException("Destination storage pools must be either all managed or all not managed"); + } + + if (!destStoragePoolVO.isManaged()) { + if (destStoragePoolVO.getPoolType() == StoragePoolType.NetworkFilesystem && + destStoragePoolVO.getScope() != ScopeType.CLUSTER) { + throw new CloudRuntimeException("KVM live storage migrations currently support cluster-wide " + + "not managed NFS destination storage"); + } + if (!sourcePools.containsKey(srcStoragePoolVO.getUuid())) { + sourcePools.put(srcStoragePoolVO.getUuid(), srcStoragePoolVO.getPoolType()); + } + } + } + verifyDestinationStorage(sourcePools, destHost); + } + + /** + * Perform storage validation on destination host for KVM live storage migrations. + * Validate that volume source storage pools are mounted on the destination host prior the migration + * @throws CloudRuntimeException if any source storage pool is not mounted on the destination host + */ + private void verifyDestinationStorage(Map sourcePools, Host destHost) { + if (MapUtils.isNotEmpty(sourcePools)) { + LOGGER.debug("Verifying source pools are already available on destination host " + destHost.getUuid()); + CheckStorageAvailabilityCommand cmd = new CheckStorageAvailabilityCommand(sourcePools); + try { + Answer answer = agentManager.send(destHost.getId(), cmd); + if (answer == null || !answer.getResult()) { + throw new CloudRuntimeException("Storage verification failed on host " + + destHost.getUuid() +": " + answer.getDetails()); + } + } catch (AgentUnavailableException | OperationTimedoutException e) { + e.printStackTrace(); + throw new CloudRuntimeException("Cannot perform storage verification on host " + destHost.getUuid() + + "due to: " + e.getMessage()); + } } } diff --git a/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageSystemDataMotionTest.java b/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageSystemDataMotionTest.java index 89a278349f2..5b8d3aff2b8 100644 --- a/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageSystemDataMotionTest.java +++ b/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageSystemDataMotionTest.java @@ -21,6 +21,10 @@ package org.apache.cloudstack.storage.motion; import java.util.HashMap; import java.util.Map; +import com.cloud.host.Host; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.storage.ScopeType; +import com.cloud.storage.Storage; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.StrategyPriority; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory; @@ -38,6 +42,7 @@ import org.apache.cloudstack.storage.image.store.ImageStoreImpl; import org.apache.cloudstack.storage.to.TemplateObjectTO; import org.apache.cloudstack.storage.volume.VolumeObject; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InOrder; @@ -53,7 +58,6 @@ import com.cloud.agent.api.MigrateCommand; import com.cloud.exception.AgentUnavailableException; import com.cloud.exception.CloudException; import com.cloud.exception.OperationTimedoutException; -import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.storage.DataStoreRole; @@ -66,6 +70,9 @@ import com.cloud.storage.dao.VMTemplatePoolDao; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VirtualMachineManager; +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.when; + @RunWith(MockitoJUnitRunner.class) public class KvmNonManagedStorageSystemDataMotionTest { @@ -90,6 +97,36 @@ public class KvmNonManagedStorageSystemDataMotionTest { @InjectMocks private KvmNonManagedStorageDataMotionStrategy kvmNonManagedStorageDataMotionStrategy; + @Mock + VolumeInfo volumeInfo1; + @Mock + VolumeInfo volumeInfo2; + @Mock + DataStore dataStore1; + @Mock + DataStore dataStore2; + @Mock + DataStore dataStore3; + @Mock + StoragePoolVO pool1; + @Mock + StoragePoolVO pool2; + @Mock + StoragePoolVO pool3; + @Mock + Host host1; + @Mock + Host host2; + + Map migrationMap; + + private static final Long POOL_1_ID = 1L; + private static final Long POOL_2_ID = 2L; + private static final Long POOL_3_ID = 3L; + private static final Long HOST_1_ID = 1L; + private static final Long HOST_2_ID = 2L; + private static final Long CLUSTER_ID = 1L; + @Test public void canHandleTestExpectHypervisorStrategyForKvm() { canHandleExpectCannotHandle(HypervisorType.KVM, 1, StrategyPriority.HYPERVISOR); @@ -109,12 +146,13 @@ public class KvmNonManagedStorageSystemDataMotionTest { private void canHandleExpectCannotHandle(HypervisorType hypervisorType, int times, StrategyPriority expectedStrategyPriority) { HostVO srcHost = new HostVO("sourceHostUuid"); + HostVO destHost = new HostVO("destHostUuid"); srcHost.setHypervisorType(hypervisorType); - Mockito.doReturn(StrategyPriority.HYPERVISOR).when(kvmNonManagedStorageDataMotionStrategy).internalCanHandle(new HashMap<>()); + Mockito.doReturn(StrategyPriority.HYPERVISOR).when(kvmNonManagedStorageDataMotionStrategy).internalCanHandle(new HashMap<>(), srcHost, destHost); - StrategyPriority strategyPriority = kvmNonManagedStorageDataMotionStrategy.canHandle(new HashMap<>(), srcHost, new HostVO("destHostUuid")); + StrategyPriority strategyPriority = kvmNonManagedStorageDataMotionStrategy.canHandle(new HashMap<>(), srcHost, destHost); - Mockito.verify(kvmNonManagedStorageDataMotionStrategy, Mockito.times(times)).internalCanHandle(new HashMap<>()); + Mockito.verify(kvmNonManagedStorageDataMotionStrategy, Mockito.times(times)).internalCanHandle(new HashMap<>(), srcHost, destHost); Assert.assertEquals(expectedStrategyPriority, strategyPriority); } @@ -123,7 +161,7 @@ public class KvmNonManagedStorageSystemDataMotionTest { StoragePoolType[] storagePoolTypeArray = StoragePoolType.values(); for (int i = 0; i < storagePoolTypeArray.length; i++) { Map volumeMap = configureTestInternalCanHandle(false, storagePoolTypeArray[i]); - StrategyPriority strategyPriority = kvmNonManagedStorageDataMotionStrategy.internalCanHandle(volumeMap); + StrategyPriority strategyPriority = kvmNonManagedStorageDataMotionStrategy.internalCanHandle(volumeMap, new HostVO("sourceHostUuid"), new HostVO("destHostUuid")); if (storagePoolTypeArray[i] == StoragePoolType.Filesystem || storagePoolTypeArray[i] == StoragePoolType.NetworkFilesystem) { Assert.assertEquals(StrategyPriority.HYPERVISOR, strategyPriority); } else { @@ -137,7 +175,7 @@ public class KvmNonManagedStorageSystemDataMotionTest { StoragePoolType[] storagePoolTypeArray = StoragePoolType.values(); for (int i = 0; i < storagePoolTypeArray.length; i++) { Map volumeMap = configureTestInternalCanHandle(true, storagePoolTypeArray[i]); - StrategyPriority strategyPriority = kvmNonManagedStorageDataMotionStrategy.internalCanHandle(volumeMap); + StrategyPriority strategyPriority = kvmNonManagedStorageDataMotionStrategy.internalCanHandle(volumeMap, null, null); Assert.assertEquals(StrategyPriority.CANT_HANDLE, strategyPriority); } } @@ -202,7 +240,7 @@ public class KvmNonManagedStorageSystemDataMotionTest { for (int i = 0; i < storagePoolTypes.length; i++) { Mockito.doReturn(storagePoolTypes[i]).when(sourceStoragePool).getPoolType(); boolean result = kvmNonManagedStorageDataMotionStrategy.shouldMigrateVolume(sourceStoragePool, destHost, destStoragePool); - if (storagePoolTypes[i] == StoragePoolType.Filesystem) { + if (storagePoolTypes[i] == StoragePoolType.Filesystem || storagePoolTypes[i] == StoragePoolType.NetworkFilesystem) { Assert.assertTrue(result); } else { Assert.assertFalse(result); @@ -330,4 +368,102 @@ public class KvmNonManagedStorageSystemDataMotionTest { verifyInOrder.verify(kvmNonManagedStorageDataMotionStrategy, Mockito.times(times)).sendCopyCommand(Mockito.eq(destHost), Mockito.any(TemplateObjectTO.class), Mockito.any(TemplateObjectTO.class), Mockito.eq(destDataStore)); } + + @Before + public void setUp() { + migrationMap = new HashMap<>(); + migrationMap.put(volumeInfo1, dataStore2); + migrationMap.put(volumeInfo2, dataStore2); + + when(volumeInfo1.getPoolId()).thenReturn(POOL_1_ID); + when(primaryDataStoreDao.findById(POOL_1_ID)).thenReturn(pool1); + when(pool1.isManaged()).thenReturn(false); + when(dataStore2.getId()).thenReturn(POOL_2_ID); + when(primaryDataStoreDao.findById(POOL_2_ID)).thenReturn(pool2); + when(pool2.isManaged()).thenReturn(true); + when(volumeInfo1.getDataStore()).thenReturn(dataStore1); + + when(volumeInfo2.getPoolId()).thenReturn(POOL_1_ID); + when(volumeInfo2.getDataStore()).thenReturn(dataStore1); + + when(dataStore1.getId()).thenReturn(POOL_1_ID); + when(pool1.getPoolType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem); + when(pool2.getPoolType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem); + when(pool2.getScope()).thenReturn(ScopeType.CLUSTER); + + when(dataStore3.getId()).thenReturn(POOL_3_ID); + when(primaryDataStoreDao.findById(POOL_3_ID)).thenReturn(pool3); + when(pool3.getPoolType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem); + when(pool3.getScope()).thenReturn(ScopeType.CLUSTER); + when(host1.getId()).thenReturn(HOST_1_ID); + when(host1.getClusterId()).thenReturn(CLUSTER_ID); + when(host1.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM); + when(host2.getId()).thenReturn(HOST_2_ID); + when(host2.getClusterId()).thenReturn(CLUSTER_ID); + when(host2.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM); + } + + @Test + public void canHandleKVMLiveStorageMigrationSameHost() { + StrategyPriority priority = kvmNonManagedStorageDataMotionStrategy.canHandleKVMNonManagedLiveNFSStorageMigration(migrationMap, host1, host1); + assertEquals(StrategyPriority.CANT_HANDLE, priority); + } + + @Test + public void canHandleKVMLiveStorageMigrationInterCluster() { + when(host2.getClusterId()).thenReturn(5L); + StrategyPriority priority = kvmNonManagedStorageDataMotionStrategy.canHandleKVMNonManagedLiveNFSStorageMigration(migrationMap, host1, host2); + assertEquals(StrategyPriority.CANT_HANDLE, priority); + } + + @Test + public void canHandleKVMLiveStorageMigration() { + StrategyPriority priority = kvmNonManagedStorageDataMotionStrategy.canHandleKVMNonManagedLiveNFSStorageMigration(migrationMap, host1, host2); + assertEquals(StrategyPriority.HYPERVISOR, priority); + } + + @Test + public void canHandleKVMLiveStorageMigrationMultipleSources() { + when(volumeInfo1.getDataStore()).thenReturn(dataStore2); + StrategyPriority priority = kvmNonManagedStorageDataMotionStrategy.canHandleKVMNonManagedLiveNFSStorageMigration(migrationMap, host1, host2); + assertEquals(StrategyPriority.HYPERVISOR, priority); + } + + @Test + public void canHandleKVMLiveStorageMigrationMultipleDestination() { + migrationMap.put(volumeInfo2, dataStore3); + StrategyPriority priority = kvmNonManagedStorageDataMotionStrategy.canHandleKVMNonManagedLiveNFSStorageMigration(migrationMap, host1, host2); + assertEquals(StrategyPriority.HYPERVISOR, priority); + } + + @Test + public void testCanHandleLiveMigrationUnmanagedStorage() { + when(pool2.isManaged()).thenReturn(false); + StrategyPriority priority = kvmNonManagedStorageDataMotionStrategy.canHandleKVMNonManagedLiveNFSStorageMigration(migrationMap, host1, host2); + assertEquals(StrategyPriority.HYPERVISOR, priority); + } + + @Test + public void testVerifyLiveMigrationMapForKVM() { + kvmNonManagedStorageDataMotionStrategy.verifyLiveMigrationForKVM(migrationMap, host2); + } + + @Test(expected = CloudRuntimeException.class) + public void testVerifyLiveMigrationMapForKVMNotExistingSource() { + when(primaryDataStoreDao.findById(POOL_1_ID)).thenReturn(null); + kvmNonManagedStorageDataMotionStrategy.verifyLiveMigrationForKVM(migrationMap, host2); + } + + @Test(expected = CloudRuntimeException.class) + public void testVerifyLiveMigrationMapForKVMNotExistingDest() { + when(primaryDataStoreDao.findById(POOL_2_ID)).thenReturn(null); + kvmNonManagedStorageDataMotionStrategy.verifyLiveMigrationForKVM(migrationMap, host2); + } + + @Test(expected = CloudRuntimeException.class) + public void testVerifyLiveMigrationMapForKVMMixedManagedUnmagedStorage() { + when(pool1.isManaged()).thenReturn(true); + when(pool2.isManaged()).thenReturn(false); + kvmNonManagedStorageDataMotionStrategy.verifyLiveMigrationForKVM(migrationMap, host2); + } } diff --git a/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategyTest.java b/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategyTest.java index 197e66ce5b0..1b383d91574 100644 --- a/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategyTest.java +++ b/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategyTest.java @@ -51,8 +51,8 @@ import com.cloud.host.HostVO; import com.cloud.storage.DataStoreRole; import com.cloud.storage.ImageStore; import com.cloud.storage.Storage; -import com.cloud.storage.Volume; import com.cloud.storage.Storage.StoragePoolType; +import com.cloud.storage.Volume; import com.cloud.storage.VolumeVO; @RunWith(MockitoJUnitRunner.class) @@ -60,14 +60,14 @@ public class StorageSystemDataMotionStrategyTest { @Spy @InjectMocks - private StorageSystemDataMotionStrategy storageSystemDataMotionStrategy; + private StorageSystemDataMotionStrategy strategy; @Mock private VolumeObject volumeObjectSource; @Mock private DataObject dataObjectDestination; @Mock - private PrimaryDataStore primaryDataStoreSourceStore; + private PrimaryDataStore sourceStore; @Mock private ImageStore destinationStore; @Mock @@ -75,26 +75,26 @@ public class StorageSystemDataMotionStrategyTest { @Before public void setUp() throws Exception { - primaryDataStoreSourceStore = mock(PrimaryDataStoreImpl.class); + sourceStore = mock(PrimaryDataStoreImpl.class); destinationStore = mock(ImageStoreImpl.class); volumeObjectSource = mock(VolumeObject.class); dataObjectDestination = mock(VolumeObject.class); - initMocks(storageSystemDataMotionStrategy); + initMocks(strategy); } @Test public void cantHandleSecondary() { - doReturn(primaryDataStoreSourceStore).when(volumeObjectSource).getDataStore(); - doReturn(DataStoreRole.Primary).when(primaryDataStoreSourceStore).getRole(); + doReturn(sourceStore).when(volumeObjectSource).getDataStore(); + doReturn(DataStoreRole.Primary).when(sourceStore).getRole(); doReturn(destinationStore).when(dataObjectDestination).getDataStore(); doReturn(DataStoreRole.Image).when((DataStore)destinationStore).getRole(); - doReturn(primaryDataStoreSourceStore).when(volumeObjectSource).getDataStore(); + doReturn(sourceStore).when(volumeObjectSource).getDataStore(); doReturn(destinationStore).when(dataObjectDestination).getDataStore(); StoragePoolVO storeVO = new StoragePoolVO(); doReturn(storeVO).when(primaryDataStoreDao).findById(0l); - assertTrue(storageSystemDataMotionStrategy.canHandle(volumeObjectSource, dataObjectDestination) == StrategyPriority.CANT_HANDLE); + assertTrue(strategy.canHandle(volumeObjectSource, dataObjectDestination) == StrategyPriority.CANT_HANDLE); } @Test @@ -135,7 +135,7 @@ public class StorageSystemDataMotionStrategyTest { Mockito.doReturn(storagePool0).when(primaryDataStoreDao).findById(0l); Mockito.doReturn(storagePool1).when(primaryDataStoreDao).findById(1l); - StrategyPriority strategyPriority = storageSystemDataMotionStrategy.internalCanHandle(volumeMap); + StrategyPriority strategyPriority = strategy.internalCanHandle(volumeMap, new HostVO("srcHostUuid"), new HostVO("destHostUuid")); Assert.assertEquals(expectedStrategyPriority, strategyPriority); } @@ -146,7 +146,7 @@ public class StorageSystemDataMotionStrategyTest { StoragePoolType[] storagePoolTypeArray = StoragePoolType.values(); for (int i = 0; i < storagePoolTypeArray.length; i++) { Mockito.doReturn(storagePoolTypeArray[i]).when(sourceStoragePool).getPoolType(); - boolean result = storageSystemDataMotionStrategy.isStoragePoolTypeOfFile(sourceStoragePool); + boolean result = strategy.isStoragePoolTypeOfFile(sourceStoragePool); if (sourceStoragePool.getPoolType() == StoragePoolType.Filesystem) { Assert.assertTrue(result); } else { @@ -161,19 +161,19 @@ public class StorageSystemDataMotionStrategyTest { HostVO destHost = new HostVO("guid"); Mockito.doReturn("iScsiName").when(destVolumeInfo).get_iScsiName(); Mockito.doReturn(0l).when(destVolumeInfo).getPoolId(); - Mockito.doReturn("expected").when(storageSystemDataMotionStrategy).connectHostToVolume(destHost, 0l, "iScsiName"); + Mockito.doReturn("expected").when(strategy).connectHostToVolume(destHost, 0l, "iScsiName"); - String expected = storageSystemDataMotionStrategy.generateDestPath(destHost, Mockito.mock(StoragePoolVO.class), destVolumeInfo); + String expected = strategy.generateDestPath(destHost, Mockito.mock(StoragePoolVO.class), destVolumeInfo); Assert.assertEquals(expected, "expected"); - Mockito.verify(storageSystemDataMotionStrategy).connectHostToVolume(destHost, 0l, "iScsiName"); + Mockito.verify(strategy).connectHostToVolume(destHost, 0l, "iScsiName"); } @Test public void configureMigrateDiskInfoTest() { VolumeObject srcVolumeInfo = Mockito.spy(new VolumeObject()); Mockito.doReturn("volume path").when(srcVolumeInfo).getPath(); - MigrateCommand.MigrateDiskInfo migrateDiskInfo = storageSystemDataMotionStrategy.configureMigrateDiskInfo(srcVolumeInfo, "destPath"); + MigrateCommand.MigrateDiskInfo migrateDiskInfo = strategy.configureMigrateDiskInfo(srcVolumeInfo, "destPath"); Assert.assertEquals(MigrateCommand.MigrateDiskInfo.DiskType.BLOCK, migrateDiskInfo.getDiskType()); Assert.assertEquals(MigrateCommand.MigrateDiskInfo.DriverType.RAW, migrateDiskInfo.getDriverType()); Assert.assertEquals(MigrateCommand.MigrateDiskInfo.Source.DEV, migrateDiskInfo.getSource()); @@ -187,7 +187,7 @@ public class StorageSystemDataMotionStrategyTest { String volumePath = "iScsiName"; volume.set_iScsiName(volumePath); - storageSystemDataMotionStrategy.setVolumePath(volume); + strategy.setVolumePath(volume); Assert.assertEquals(volumePath, volume.getPath()); } @@ -200,8 +200,9 @@ public class StorageSystemDataMotionStrategyTest { StoragePoolType[] storagePoolTypes = StoragePoolType.values(); for (int i = 0; i < storagePoolTypes.length; i++) { Mockito.doReturn(storagePoolTypes[i]).when(sourceStoragePool).getPoolType(); - boolean result = storageSystemDataMotionStrategy.shouldMigrateVolume(sourceStoragePool, destHost, destStoragePool); + boolean result = strategy.shouldMigrateVolume(sourceStoragePool, destHost, destStoragePool); Assert.assertTrue(result); } } -} + +} \ No newline at end of file diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java index 45e3941a5ec..097b85477da 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java @@ -409,8 +409,15 @@ public class TemplateServiceImpl implements TemplateService { _templateDao.update(tmplt.getId(), tmlpt); if (tmplt.getState() == VirtualMachineTemplate.State.NotUploaded || tmplt.getState() == VirtualMachineTemplate.State.UploadInProgress) { + VirtualMachineTemplate.Event event = VirtualMachineTemplate.Event.OperationSucceeded; + // For multi-disk OVA, check and create data disk templates + if (tmplt.getFormat().equals(ImageFormat.OVA)) { + if (!createOvaDataDiskTemplates(_templateFactory.getTemplate(tmlpt.getId(), store))) { + event = VirtualMachineTemplate.Event.OperationFailed; + } + } try { - stateMachine.transitTo(tmplt, VirtualMachineTemplate.Event.OperationSucceeded, null, _templateDao); + stateMachine.transitTo(tmplt, event, null, _templateDao); } catch (NoTransitionException e) { s_logger.error("Unexpected state transition exception for template " + tmplt.getName() + ". Details: " + e.getMessage()); } @@ -701,7 +708,7 @@ public class TemplateServiceImpl implements TemplateService { return null; } - // Check if OVA contains additional data disks. If yes, create Datadisk templates for each of the additional datadisk present in the OVA + // For multi-disk OVA, check and create data disk templates if (template.getFormat().equals(ImageFormat.OVA)) { if (!createOvaDataDiskTemplates(template)) { template.processEvent(ObjectInDataStoreStateMachine.Event.OperationFailed); @@ -729,8 +736,8 @@ public class TemplateServiceImpl implements TemplateService { return null; } - - protected boolean createOvaDataDiskTemplates(TemplateInfo parentTemplate) { + @Override + public boolean createOvaDataDiskTemplates(TemplateInfo parentTemplate) { try { // Get Datadisk template (if any) for OVA List dataDiskTemplates = new ArrayList(); diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java index 5117b7cb84f..062e89a4247 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java @@ -90,6 +90,7 @@ public class ObjectInDataStoreManagerImpl implements ObjectInDataStoreManager { stateMachines.addTransition(State.Allocated, Event.CreateOnlyRequested, State.Creating); stateMachines.addTransition(State.Allocated, Event.DestroyRequested, State.Destroying); stateMachines.addTransition(State.Allocated, Event.OperationFailed, State.Failed); + stateMachines.addTransition(State.Allocated, Event.OperationSuccessed, State.Ready); stateMachines.addTransition(State.Creating, Event.OperationFailed, State.Allocated); stateMachines.addTransition(State.Creating, Event.OperationSuccessed, State.Ready); stateMachines.addTransition(State.Ready, Event.CopyingRequested, State.Copying); diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java index 85d95240617..d62a0baa04a 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java @@ -20,6 +20,7 @@ import java.util.Date; import javax.inject.Inject; +import com.cloud.storage.MigrationOptions; import org.apache.log4j.Logger; import org.apache.cloudstack.engine.subsystem.api.storage.DataObjectInStore; @@ -72,6 +73,7 @@ public class VolumeObject implements VolumeInfo { @Inject DiskOfferingDao diskOfferingDao; private Object payload; + private MigrationOptions migrationOptions; public VolumeObject() { _volStateMachine = Volume.State.getStateMachine(); @@ -315,6 +317,16 @@ public class VolumeObject implements VolumeInfo { return null; } + @Override + public MigrationOptions getMigrationOptions() { + return migrationOptions; + } + + @Override + public void setMigrationOptions(MigrationOptions migrationOptions) { + this.migrationOptions = migrationOptions; + } + public void update() { volumeDao.update(volumeVO.getId(), volumeVO); volumeVO = volumeDao.findById(volumeVO.getId()); diff --git a/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/manager/BareMetalTemplateAdapter.java b/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/manager/BareMetalTemplateAdapter.java index d2d0029701a..8265f951f8a 100644 --- a/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/manager/BareMetalTemplateAdapter.java +++ b/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/manager/BareMetalTemplateAdapter.java @@ -40,6 +40,7 @@ import com.cloud.user.Account; import com.cloud.utils.db.DB; import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.api.command.user.iso.DeleteIsoCmd; +import org.apache.cloudstack.api.command.user.iso.GetUploadParamsForIsoCmd; import org.apache.cloudstack.api.command.user.iso.RegisterIsoCmd; import org.apache.cloudstack.api.command.user.template.RegisterTemplateCmd; import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand; @@ -72,6 +73,11 @@ public class BareMetalTemplateAdapter extends TemplateAdapterBase implements Tem throw new CloudRuntimeException("Baremetal doesn't support ISO template"); } + @Override + public TemplateProfile prepare(GetUploadParamsForIsoCmd cmd) throws ResourceAllocationException { + throw new CloudRuntimeException("Baremetal doesn't support ISO template"); + } + private void templateCreateUsage(VMTemplateVO template, long dcId) { if (template.getAccountId() != Account.ACCOUNT_ID_SYSTEM) { UsageEventVO usageEvent = diff --git a/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalDhcpElement.java b/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalDhcpElement.java index 5696048b847..807babcb09f 100644 --- a/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalDhcpElement.java +++ b/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalDhcpElement.java @@ -185,4 +185,9 @@ public class BaremetalDhcpElement extends AdapterBase implements DhcpServiceProv return false; } + @Override + public boolean removeDhcpEntry(Network network, NicProfile nic, VirtualMachineProfile vmProfile) throws ResourceUnavailableException { + return false; + } + } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/MigrateKVMAsync.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/MigrateKVMAsync.java index 51dbd9234ac..c3e3e6edf41 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/MigrateKVMAsync.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/MigrateKVMAsync.java @@ -34,46 +34,67 @@ public class MigrateKVMAsync implements Callable { private String vmName = ""; private String destIp = ""; private boolean migrateStorage; + private boolean migrateStorageManaged; private boolean autoConvergence; - /** - * Do not pause the domain during migration. The domain's memory will be transferred to the destination host while the domain is running. The migration may never converge if the domain is changing its memory faster then it can be transferred. The domain can be manually paused anytime during migration using virDomainSuspend. - * @value 1 - * @see Libvirt virDomainMigrateFlags documentation - */ + // Libvirt Migrate Flags reference: + // https://libvirt.org/html/libvirt-libvirt-domain.html#virDomainMigrateFlags + + // Do not pause the domain during migration. The domain's memory will be + // transferred to the destination host while the domain is running. The migration + // may never converge if the domain is changing its memory faster then it can be + // transferred. The domain can be manually paused anytime during migration using + // virDomainSuspend. private static final long VIR_MIGRATE_LIVE = 1L; - /** - * Migrate full disk images in addition to domain's memory. By default only non-shared non-readonly disk images are transferred. The VIR_MIGRATE_PARAM_MIGRATE_DISKS parameter can be used to specify which disks should be migrated. This flag and VIR_MIGRATE_NON_SHARED_INC are mutually exclusive. - * @value 64 - * @see Libvirt virDomainMigrateFlags documentation - */ + + // Define the domain as persistent on the destination host after successful + // migration. If the domain was persistent on the source host and + // VIR_MIGRATE_UNDEFINE_SOURCE is not used, it will end up persistent on both + // hosts. + private static final long VIR_MIGRATE_PERSIST_DEST = 8L; + + // Migrate full disk images in addition to domain's memory. By default only + // non-shared non-readonly disk images are transferred. The + // VIR_MIGRATE_PARAM_MIGRATE_DISKS parameter can be used to specify which disks + // should be migrated. This flag and VIR_MIGRATE_NON_SHARED_INC are mutually + // exclusive. private static final long VIR_MIGRATE_NON_SHARED_DISK = 64L; - /** - * Compress migration data. The compression methods can be specified using VIR_MIGRATE_PARAM_COMPRESSION. A hypervisor default method will be used if this parameter is omitted. Individual compression methods can be tuned via their specific VIR_MIGRATE_PARAM_COMPRESSION_* parameters. - * @value 2048 - * @see Libvirt virDomainMigrateFlags documentation - */ + + // Migrate disk images in addition to domain's memory. This is similar to + // VIR_MIGRATE_NON_SHARED_DISK, but only the top level of each disk's backing chain + // is copied. That is, the rest of the backing chain is expected to be present on + // the destination and to be exactly the same as on the source host. This flag and + // VIR_MIGRATE_NON_SHARED_DISK are mutually exclusive. + private static final long VIR_MIGRATE_NON_SHARED_INC = 128L; + + // Compress migration data. The compression methods can be specified using + // VIR_MIGRATE_PARAM_COMPRESSION. A hypervisor default method will be used if this + // parameter is omitted. Individual compression methods can be tuned via their + // specific VIR_MIGRATE_PARAM_COMPRESSION_* parameters. private static final long VIR_MIGRATE_COMPRESSED = 2048L; - /** - * Enable algorithms that ensure a live migration will eventually converge. This usually means the domain will be slowed down to make sure it does not change its memory faster than a hypervisor can transfer the changed memory to the destination host. VIR_MIGRATE_PARAM_AUTO_CONVERGE_* parameters can be used to tune the algorithm. - * @value 8192 - * @see Libvirt virDomainMigrateFlags documentation - */ + + // Enable algorithms that ensure a live migration will eventually converge. + // This usually means the domain will be slowed down to make sure it does not + // change its memory faster than a hypervisor can transfer the changed memory to + // the destination host. VIR_MIGRATE_PARAM_AUTO_CONVERGE_* parameters can be used + // to tune the algorithm. private static final long VIR_MIGRATE_AUTO_CONVERGE = 8192L; - /** - * Libvirt 1.0.3 supports compression flag for migration. - */ + // Libvirt 1.0.3 supports compression flag for migration. private static final int LIBVIRT_VERSION_SUPPORTS_MIGRATE_COMPRESSED = 1000003; + // Libvirt 1.2.3 supports auto converge. + private static final int LIBVIRT_VERSION_SUPPORTS_AUTO_CONVERGE = 1002003; + public MigrateKVMAsync(final LibvirtComputingResource libvirtComputingResource, final Domain dm, final Connect dconn, final String dxml, - final boolean migrateStorage, final boolean autoConvergence, final String vmName, final String destIp) { + final boolean migrateStorage, final boolean migrateStorageManaged, final boolean autoConvergence, final String vmName, final String destIp) { this.libvirtComputingResource = libvirtComputingResource; this.dm = dm; this.dconn = dconn; this.dxml = dxml; this.migrateStorage = migrateStorage; + this.migrateStorageManaged = migrateStorageManaged; this.autoConvergence = autoConvergence; this.vmName = vmName; this.destIp = destIp; @@ -84,15 +105,20 @@ public class MigrateKVMAsync implements Callable { long flags = VIR_MIGRATE_LIVE; if (dconn.getLibVirVersion() >= LIBVIRT_VERSION_SUPPORTS_MIGRATE_COMPRESSED) { - flags += VIR_MIGRATE_COMPRESSED; + flags |= VIR_MIGRATE_COMPRESSED; } if (migrateStorage) { - flags += VIR_MIGRATE_NON_SHARED_DISK; + if (migrateStorageManaged) { + flags |= VIR_MIGRATE_NON_SHARED_DISK; + } else { + flags |= VIR_MIGRATE_PERSIST_DEST; + flags |= VIR_MIGRATE_NON_SHARED_INC; + } } - if (autoConvergence && dconn.getLibVirVersion() >= 1002003) { - flags += VIR_MIGRATE_AUTO_CONVERGE; + if (autoConvergence && dconn.getLibVirVersion() >= LIBVIRT_VERSION_SUPPORTS_AUTO_CONVERGE) { + flags |= VIR_MIGRATE_AUTO_CONVERGE; } return dm.migrate(dconn, flags, dxml, vmName, "tcp:" + destIp, libvirtComputingResource.getMigrateSpeed()); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckStorageAvailabilityWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckStorageAvailabilityWrapper.java new file mode 100644 index 00000000000..5022e01f21c --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckStorageAvailabilityWrapper.java @@ -0,0 +1,61 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.storage.CheckStorageAvailabilityCommand; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.storage.KVMStoragePool; +import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.storage.Storage; +import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.log4j.Logger; + +import java.util.Map; + +@ResourceWrapper(handles = CheckStorageAvailabilityCommand.class) +public class LibvirtCheckStorageAvailabilityWrapper extends CommandWrapper { + + private static final Logger s_logger = Logger.getLogger(LibvirtCheckStorageAvailabilityWrapper.class); + + @Override + public Answer execute(CheckStorageAvailabilityCommand command, LibvirtComputingResource resource) { + KVMStoragePoolManager storagePoolMgr = resource.getStoragePoolMgr(); + Map poolsMap = command.getPoolsMap(); + + for (String poolUuid : poolsMap.keySet()) { + Storage.StoragePoolType type = poolsMap.get(poolUuid); + s_logger.debug("Checking if storage pool " + poolUuid + " (" + type + ") is mounted on this host"); + try { + KVMStoragePool storagePool = storagePoolMgr.getStoragePool(type, poolUuid); + if (storagePool == null) { + s_logger.info("Storage pool " + poolUuid + " is not available"); + return new Answer(command, false, "Storage pool " + poolUuid + " not available"); + } + } catch (CloudRuntimeException e) { + s_logger.info("Storage pool " + poolUuid + " is not available"); + return new Answer(command, e); + } + } + return new Answer(command); + } +} \ No newline at end of file diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java index 0c1370e2551..5bf8d25e949 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java @@ -147,9 +147,10 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper worker = new MigrateKVMAsync(libvirtComputingResource, dm, dconn, xmlDesc, migrateStorage, + final Callable worker = new MigrateKVMAsync(libvirtComputingResource, dm, dconn, xmlDesc, + migrateStorage, migrateStorageManaged, command.isAutoConvergence(), vmName, command.getDestinationIp()); final Future migrateThread = executor.submit(worker); executor.shutdown(); @@ -356,7 +358,8 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapperThe source of the disk needs an attribute that is either 'file' or 'dev' as well as its corresponding value. *
*/ - protected String replaceStorage(String xmlDesc, Map migrateStorage) + protected String replaceStorage(String xmlDesc, Map migrateStorage, + boolean migrateStorageManaged) throws IOException, ParserConfigurationException, SAXException, TransformerException { InputStream in = IOUtils.toInputStream(xmlDesc); @@ -398,7 +401,7 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper { + + private static final String temporaryCertFilePrefix = "CSCERTIFICATE"; + + private static final Logger s_logger = Logger.getLogger(LibvirtSetupDirectDownloadCertificateCommandWrapper.class); + + /** + * Retrieve agent.properties file + */ + private File getAgentPropertiesFile() throws FileNotFoundException { + final File agentFile = PropertiesUtil.findConfigFile("agent.properties"); + if (agentFile == null) { + throw new FileNotFoundException("Failed to find agent.properties file"); + } + return agentFile; + } + + /** + * Get the property 'keystore.passphrase' value from agent.properties file + */ + private String getKeystorePassword(File agentFile) { + String pass = null; + if (agentFile != null) { + try { + pass = PropertiesUtil.loadFromFile(agentFile).getProperty(KeyStoreUtils.KS_PASSPHRASE_PROPERTY); + } catch (IOException e) { + s_logger.error("Could not get 'keystore.passphrase' property value due to: " + e.getMessage()); + } + } + return pass; + } + + /** + * Get keystore path + */ + private String getKeyStoreFilePath(File agentFile) { + return agentFile.getParent() + "/" + KeyStoreUtils.KS_FILENAME; + } + + /** + * Import certificate from temporary file into keystore + */ + private void importCertificate(String tempCerFilePath, String keyStoreFile, String certificateName, String privatePassword) { + s_logger.debug("Importing certificate from temporary file to keystore"); + String importCommandFormat = "keytool -importcert -file %s -keystore %s -alias '%s' -storepass '%s' -noprompt"; + String importCmd = String.format(importCommandFormat, tempCerFilePath, keyStoreFile, certificateName, privatePassword); + int result = Script.runSimpleBashScriptForExitValue(importCmd); + if (result != 0) { + s_logger.debug("Certificate " + certificateName + " not imported as it already exist on keystore"); + } + } + + /** + * Create temporary file and return its path + */ + private String createTemporaryFile(File agentFile, String certificateName, String certificate) { + String tempCerFilePath = String.format("%s/%s-%s", + agentFile.getParent(), temporaryCertFilePrefix, certificateName); + s_logger.debug("Creating temporary certificate file into: " + tempCerFilePath); + int result = Script.runSimpleBashScriptForExitValue(String.format("echo '%s' > %s", certificate, tempCerFilePath)); + if (result != 0) { + throw new CloudRuntimeException("Could not create the certificate file on path: " + tempCerFilePath); + } + return tempCerFilePath; + } + + /** + * Remove temporary file + */ + private void cleanupTemporaryFile(String temporaryFile) { + s_logger.debug("Cleaning up temporary certificate file"); + Script.runSimpleBashScript("rm -f " + temporaryFile); + } + + @Override + public Answer execute(SetupDirectDownloadCertificateCommand cmd, LibvirtComputingResource serverResource) { + String certificate = cmd.getCertificate(); + String certificateName = cmd.getCertificateName(); + + try { + File agentFile = getAgentPropertiesFile(); + String privatePassword = getKeystorePassword(agentFile); + if (isBlank(privatePassword)) { + return new Answer(cmd, false, "No password found for keystore: " + KeyStoreUtils.KS_FILENAME); + } + + final String keyStoreFile = getKeyStoreFilePath(agentFile); + String temporaryFile = createTemporaryFile(agentFile, certificateName, certificate); + importCertificate(temporaryFile, keyStoreFile, certificateName, privatePassword); + cleanupTemporaryFile(temporaryFile); + } catch (FileNotFoundException | CloudRuntimeException e) { + s_logger.error("Error while setting up certificate " + certificateName, e); + return new Answer(cmd, false, e.getMessage()); + } + + return new Answer(cmd, true, "Certificate " + certificateName + " imported"); + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/IscsiAdmStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/IscsiAdmStorageAdaptor.java index efb51fb82cc..8d1ae771b29 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/IscsiAdmStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/IscsiAdmStorageAdaptor.java @@ -427,7 +427,7 @@ public class IscsiAdmStorageAdaptor implements StorageAdaptor { } @Override - public KVMPhysicalDisk createDiskFromSnapshot(KVMPhysicalDisk snapshot, String snapshotName, String name, KVMStoragePool destPool) { + public KVMPhysicalDisk createDiskFromSnapshot(KVMPhysicalDisk snapshot, String snapshotName, String name, KVMStoragePool destPool, int timeout) { throw new UnsupportedOperationException("Creating a disk from a snapshot is not supported in this configuration."); } @@ -440,4 +440,9 @@ public class IscsiAdmStorageAdaptor implements StorageAdaptor { public boolean createFolder(String uuid, String path) { throw new UnsupportedOperationException("A folder cannot be created in this configuration."); } + + @Override + public KVMPhysicalDisk createDiskFromTemplateBacking(KVMPhysicalDisk template, String name, PhysicalDiskFormat format, long size, KVMStoragePool destPool, int timeout) { + return null; + } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java index 91cfc4ef582..c1f73d7a088 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java @@ -394,9 +394,15 @@ public class KVMStoragePoolManager { return adaptor.copyPhysicalDisk(disk, name, destPool, timeout); } - public KVMPhysicalDisk createDiskFromSnapshot(KVMPhysicalDisk snapshot, String snapshotName, String name, KVMStoragePool destPool) { + public KVMPhysicalDisk createDiskFromSnapshot(KVMPhysicalDisk snapshot, String snapshotName, String name, KVMStoragePool destPool, int timeout) { StorageAdaptor adaptor = getStorageAdaptor(destPool.getType()); - return adaptor.createDiskFromSnapshot(snapshot, snapshotName, name, destPool); + return adaptor.createDiskFromSnapshot(snapshot, snapshotName, name, destPool, timeout); + } + + public KVMPhysicalDisk createDiskWithTemplateBacking(KVMPhysicalDisk template, String name, PhysicalDiskFormat format, long size, + KVMStoragePool destPool, int timeout) { + StorageAdaptor adaptor = getStorageAdaptor(destPool.getType()); + return adaptor.createDiskFromTemplateBacking(template, name, format, size, destPool, timeout); } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index 83a7a12d22b..9a2fd275dd3 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -36,19 +36,12 @@ import java.util.UUID; import javax.naming.ConfigurationException; -import com.cloud.agent.direct.download.DirectTemplateDownloader; -import com.cloud.agent.direct.download.DirectTemplateDownloader.DirectTemplateInformation; -import com.cloud.agent.direct.download.HttpDirectTemplateDownloader; -import com.cloud.agent.direct.download.MetalinkDirectTemplateDownloader; -import com.cloud.agent.direct.download.NfsDirectTemplateDownloader; -import com.cloud.agent.direct.download.HttpsDirectTemplateDownloader; -import com.cloud.exception.InvalidParameterValueException; -import org.apache.cloudstack.agent.directdownload.HttpsDirectDownloadCommand; +import org.apache.cloudstack.agent.directdownload.DirectDownloadAnswer; import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand; import org.apache.cloudstack.agent.directdownload.HttpDirectDownloadCommand; +import org.apache.cloudstack.agent.directdownload.HttpsDirectDownloadCommand; import org.apache.cloudstack.agent.directdownload.MetalinkDirectDownloadCommand; import org.apache.cloudstack.agent.directdownload.NfsDirectDownloadCommand; -import org.apache.cloudstack.agent.directdownload.DirectDownloadAnswer; import org.apache.cloudstack.storage.command.AttachAnswer; import org.apache.cloudstack.storage.command.AttachCommand; import org.apache.cloudstack.storage.command.CopyCmdAnswer; @@ -95,7 +88,14 @@ import com.cloud.agent.api.to.DataTO; import com.cloud.agent.api.to.DiskTO; import com.cloud.agent.api.to.NfsTO; import com.cloud.agent.api.to.S3TO; +import com.cloud.agent.direct.download.DirectTemplateDownloader; +import com.cloud.agent.direct.download.DirectTemplateDownloader.DirectTemplateInformation; +import com.cloud.agent.direct.download.HttpDirectTemplateDownloader; +import com.cloud.agent.direct.download.HttpsDirectTemplateDownloader; +import com.cloud.agent.direct.download.MetalinkDirectTemplateDownloader; +import com.cloud.agent.direct.download.NfsDirectTemplateDownloader; import com.cloud.exception.InternalErrorException; +import com.cloud.exception.InvalidParameterValueException; import com.cloud.hypervisor.Hypervisor; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; import com.cloud.hypervisor.kvm.resource.LibvirtConnection; @@ -105,6 +105,7 @@ import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.DiskDef.DeviceType; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.DiskDef.DiscardType; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.DiskDef.DiskProtocol; import com.cloud.storage.JavaStorageLayer; +import com.cloud.storage.MigrationOptions; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.Storage.StoragePoolType; import com.cloud.storage.StorageLayer; @@ -114,9 +115,9 @@ import com.cloud.storage.template.Processor.FormatInfo; import com.cloud.storage.template.QCOW2Processor; import com.cloud.storage.template.TemplateLocation; import com.cloud.utils.NumbersUtil; -import com.cloud.utils.storage.S3.S3Utils; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.Script; +import com.cloud.utils.storage.S3.S3Utils; public class KVMStorageProcessor implements StorageProcessor { private static final Logger s_logger = Logger.getLogger(KVMStorageProcessor.class); @@ -1352,6 +1353,32 @@ public class KVMStorageProcessor implements StorageProcessor { } } + /** + * Create volume with backing file (linked clone) + */ + protected KVMPhysicalDisk createLinkedCloneVolume(MigrationOptions migrationOptions, KVMStoragePool srcPool, KVMStoragePool primaryPool, VolumeObjectTO volume, PhysicalDiskFormat format, int timeout) { + String srcBackingFilePath = migrationOptions.getSrcBackingFilePath(); + boolean copySrcTemplate = migrationOptions.isCopySrcTemplate(); + KVMPhysicalDisk srcTemplate = srcPool.getPhysicalDisk(srcBackingFilePath); + KVMPhysicalDisk destTemplate; + if (copySrcTemplate) { + KVMPhysicalDisk copiedTemplate = storagePoolMgr.copyPhysicalDisk(srcTemplate, srcTemplate.getName(), primaryPool, 10000 * 1000); + destTemplate = primaryPool.getPhysicalDisk(copiedTemplate.getPath()); + } else { + destTemplate = primaryPool.getPhysicalDisk(srcBackingFilePath); + } + return storagePoolMgr.createDiskWithTemplateBacking(destTemplate, volume.getUuid(), format, volume.getSize(), + primaryPool, timeout); + } + + /** + * Create full clone volume from VM snapshot + */ + protected KVMPhysicalDisk createFullCloneVolume(MigrationOptions migrationOptions, VolumeObjectTO volume, KVMStoragePool primaryPool, PhysicalDiskFormat format) { + s_logger.debug("For VM migration with full-clone volume: Creating empty stub disk for source disk " + migrationOptions.getSrcVolumeUuid() + " and size: " + volume.getSize() + " and format: " + format); + return primaryPool.createPhysicalDisk(volume.getUuid(), format, volume.getProvisioningType(), volume.getSize()); + } + @Override public Answer createVolume(final CreateObjectCommand cmd) { final VolumeObjectTO volume = (VolumeObjectTO)cmd.getData(); @@ -1369,8 +1396,23 @@ public class KVMStorageProcessor implements StorageProcessor { } else { format = PhysicalDiskFormat.valueOf(volume.getFormat().toString().toUpperCase()); } - vol = primaryPool.createPhysicalDisk(volume.getUuid(), format, - volume.getProvisioningType(), disksize); + + MigrationOptions migrationOptions = volume.getMigrationOptions(); + if (migrationOptions != null) { + String srcStoreUuid = migrationOptions.getSrcPoolUuid(); + StoragePoolType srcPoolType = migrationOptions.getSrcPoolType(); + KVMStoragePool srcPool = storagePoolMgr.getStoragePool(srcPoolType, srcStoreUuid); + int timeout = migrationOptions.getTimeout(); + + if (migrationOptions.getType() == MigrationOptions.Type.LinkedClone) { + vol = createLinkedCloneVolume(migrationOptions, srcPool, primaryPool, volume, format, timeout); + } else if (migrationOptions.getType() == MigrationOptions.Type.FullClone) { + vol = createFullCloneVolume(migrationOptions, volume, primaryPool, format); + } + } else { + vol = primaryPool.createPhysicalDisk(volume.getUuid(), format, + volume.getProvisioningType(), disksize); + } final VolumeObjectTO newVol = new VolumeObjectTO(); if(vol != null) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java index dc8083f0eea..f858a4f1577 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java @@ -95,6 +95,33 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { return true; } + @Override + public KVMPhysicalDisk createDiskFromTemplateBacking(KVMPhysicalDisk template, String name, PhysicalDiskFormat format, long size, + KVMStoragePool destPool, int timeout) { + s_logger.info("Creating volume " + name + " with template backing " + template.getName() + " in pool " + destPool.getUuid() + + " (" + destPool.getType().toString() + ") with size " + size); + + KVMPhysicalDisk disk = null; + String destPath = destPool.getLocalPath().endsWith("/") ? + destPool.getLocalPath() + name : + destPool.getLocalPath() + "/" + name; + + if (destPool.getType() == StoragePoolType.NetworkFilesystem) { + try { + if (format == PhysicalDiskFormat.QCOW2) { + QemuImg qemu = new QemuImg(timeout); + QemuImgFile destFile = new QemuImgFile(destPath, format); + destFile.setSize(size); + QemuImgFile backingFile = new QemuImgFile(template.getPath(), template.getFormat()); + qemu.create(destFile, backingFile); + } + } catch (QemuImgException e) { + s_logger.error("Failed to create " + destPath + " due to a failed executing of qemu-img: " + e.getMessage()); + } + } + return disk; + } + public StorageVol getVolume(StoragePool pool, String volName) { StorageVol vol = null; @@ -914,7 +941,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { case SPARSE: case FAT: QemuImgFile srcFile = new QemuImgFile(template.getPath(), template.getFormat()); - qemu.convert(srcFile, destFile, options); + qemu.convert(srcFile, destFile, options, null); break; } } else if (format == PhysicalDiskFormat.RAW) { @@ -927,7 +954,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { } QemuImg qemu = new QemuImg(timeout); Map options = new HashMap(); - qemu.convert(sourceFile, destFile, options); + qemu.convert(sourceFile, destFile, options, null); } } catch (QemuImgException e) { s_logger.error("Failed to create " + disk.getPath() + @@ -1302,8 +1329,35 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { } @Override - public KVMPhysicalDisk createDiskFromSnapshot(KVMPhysicalDisk snapshot, String snapshotName, String name, KVMStoragePool destPool) { - return null; + public KVMPhysicalDisk createDiskFromSnapshot(KVMPhysicalDisk snapshot, String snapshotName, String name, KVMStoragePool destPool, int timeout) { + s_logger.info("Creating volume " + name + " from snapshot " + snapshotName + " in pool " + destPool.getUuid() + + " (" + destPool.getType().toString() + ")"); + + PhysicalDiskFormat format = snapshot.getFormat(); + long size = snapshot.getSize(); + String destPath = destPool.getLocalPath().endsWith("/") ? + destPool.getLocalPath() + name : + destPool.getLocalPath() + "/" + name; + + if (destPool.getType() == StoragePoolType.NetworkFilesystem) { + try { + if (format == PhysicalDiskFormat.QCOW2) { + QemuImg qemu = new QemuImg(timeout); + QemuImgFile destFile = new QemuImgFile(destPath, format); + if (size > snapshot.getVirtualSize()) { + destFile.setSize(size); + } else { + destFile.setSize(snapshot.getVirtualSize()); + } + QemuImgFile srcFile = new QemuImgFile(snapshot.getPath(), snapshot.getFormat()); + qemu.convert(srcFile, destFile, snapshotName); + } + } catch (QemuImgException e) { + s_logger.error("Failed to create " + destPath + + " due to a failed executing of qemu-img: " + e.getMessage()); + } + } + return destPool.getPhysicalDisk(name); } @Override diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ManagedNfsStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ManagedNfsStorageAdaptor.java index 596582db34d..309308ae9c1 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ManagedNfsStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ManagedNfsStorageAdaptor.java @@ -294,7 +294,7 @@ public class ManagedNfsStorageAdaptor implements StorageAdaptor { } @Override - public KVMPhysicalDisk createDiskFromSnapshot(KVMPhysicalDisk snapshot, String snapshotName, String name, KVMStoragePool destPool) { + public KVMPhysicalDisk createDiskFromSnapshot(KVMPhysicalDisk snapshot, String snapshotName, String name, KVMStoragePool destPool, int timeout) { throw new UnsupportedOperationException("Creating a disk from a snapshot is not supported in this configuration."); } @@ -313,6 +313,11 @@ public class ManagedNfsStorageAdaptor implements StorageAdaptor { return true; } + @Override + public KVMPhysicalDisk createDiskFromTemplateBacking(KVMPhysicalDisk template, String name, PhysicalDiskFormat format, long size, KVMStoragePool destPool, int timeout) { + return null; + } + @Override public KVMPhysicalDisk createPhysicalDisk(String name, KVMStoragePool pool, PhysicalDiskFormat format, ProvisioningType provisioningType, long size) { return null; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/StorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/StorageAdaptor.java index 2c1ed233b40..a3c1387aa6b 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/StorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/StorageAdaptor.java @@ -66,11 +66,19 @@ public interface StorageAdaptor { public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMStoragePool destPools, int timeout); - public KVMPhysicalDisk createDiskFromSnapshot(KVMPhysicalDisk snapshot, String snapshotName, String name, KVMStoragePool destPool); + public KVMPhysicalDisk createDiskFromSnapshot(KVMPhysicalDisk snapshot, String snapshotName, String name, KVMStoragePool destPool, int timeout); public boolean refresh(KVMStoragePool pool); public boolean deleteStoragePool(KVMStoragePool pool); public boolean createFolder(String uuid, String path); + + /** + * Creates disk using template backing. + * Precondition: Template is on destPool + */ + KVMPhysicalDisk createDiskFromTemplateBacking(KVMPhysicalDisk template, + String name, PhysicalDiskFormat format, long size, + KVMStoragePool destPool, int timeout); } diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImg.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImg.java index eb1eeea2546..481dcdcd406 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImg.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImg.java @@ -20,17 +20,24 @@ import java.util.HashMap; import java.util.Iterator; import java.util.Map; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.NotImplementedException; + import com.cloud.storage.Storage; import com.cloud.utils.script.OutputInterpreter; import com.cloud.utils.script.Script; -import org.apache.commons.lang.NotImplementedException; public class QemuImg { /* The qemu-img binary. We expect this to be in $PATH */ public String _qemuImgPath = "qemu-img"; + private String cloudQemuImgPath = "cloud-qemu-img"; private int timeout; + private String getQemuImgPathScript = String.format("which %s >& /dev/null; " + + "if [ $? -gt 0 ]; then echo \"%s\"; else echo \"%s\"; fi", + cloudQemuImgPath, _qemuImgPath, cloudQemuImgPath); + /* Shouldn't we have KVMPhysicalDisk and LibvirtVMDef read this? */ public static enum PhysicalDiskFormat { RAW("raw"), QCOW2("qcow2"), VMDK("vmdk"), FILE("file"), RBD("rbd"), SHEEPDOG("sheepdog"), HTTP("http"), HTTPS("https"), TAR("tar"), DIR("dir"); @@ -220,10 +227,18 @@ public class QemuImg { * @param options * Options for the convert. Takes a Map with key value * pairs which are passed on to qemu-img without validation. + * @param snapshotName + * If it is provided, convertion uses it as parameter * @return void */ - public void convert(final QemuImgFile srcFile, final QemuImgFile destFile, final Map options) throws QemuImgException { - final Script script = new Script(_qemuImgPath, timeout); + public void convert(final QemuImgFile srcFile, final QemuImgFile destFile, + final Map options, final String snapshotName) throws QemuImgException { + Script script = new Script(_qemuImgPath, timeout); + if (StringUtils.isNotBlank(snapshotName)) { + String qemuPath = Script.runSimpleBashScript(getQemuImgPathScript); + script = new Script(qemuPath, timeout); + } + script.add("convert"); // autodetect source format. Sometime int he future we may teach KVMPhysicalDisk about more formats, then we can explicitly pass them if necessary //s.add("-f"); @@ -242,6 +257,13 @@ public class QemuImg { script.add(optionsStr); } + if (StringUtils.isNotBlank(snapshotName)) { + script.add("-f"); + script.add(srcFile.getFormat().toString()); + script.add("-s"); + script.add(snapshotName); + } + script.add(srcFile.getFileName()); script.add(destFile.getFileName()); @@ -269,7 +291,26 @@ public class QemuImg { * @return void */ public void convert(final QemuImgFile srcFile, final QemuImgFile destFile) throws QemuImgException { - this.convert(srcFile, destFile, null); + this.convert(srcFile, destFile, null, null); + } + + /** + * Convert a image from source to destination + * + * This method calls 'qemu-img convert' and takes three objects + * as an argument. + * + * + * @param srcFile + * The source file + * @param destFile + * The destination file + * @param snapshotName + * The snapshot name + * @return void + */ + public void convert(final QemuImgFile srcFile, final QemuImgFile destFile, String snapshotName) throws QemuImgException { + this.convert(srcFile, destFile, null, snapshotName); } /** diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapperTest.java index eaab179726a..5289481b66d 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapperTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapperTest.java @@ -19,16 +19,22 @@ package com.cloud.hypervisor.kvm.resource.wrapper; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import java.io.InputStream; +import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Scanner; import org.apache.cloudstack.utils.linux.MemStat; import java.util.Map; -import java.util.HashMap; import org.apache.commons.io.IOUtils; + +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.TransformerException; + import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -40,7 +46,9 @@ import org.mockito.Mockito; import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; +import org.xml.sax.SAXException; +import com.cloud.agent.api.MigrateCommand; import com.cloud.agent.api.MigrateCommand.MigrateDiskInfo; import com.cloud.agent.api.MigrateCommand.MigrateDiskInfo.DiskType; import com.cloud.agent.api.MigrateCommand.MigrateDiskInfo.DriverType; @@ -310,6 +318,114 @@ public class LibvirtMigrateCommandWrapperTest { PowerMockito.whenNew(Scanner.class).withAnyArguments().thenReturn(scanner); } + private static final String sourcePoolUuid = "07eb495b-5590-3877-9fb7-23c6e9a40d40"; + private static final String destPoolUuid = "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + private static final String disk1SourceFilename = "981ab1dc-40f4-41b5-b387-6539aeddbf47"; + private static final String disk2SourceFilename = "bf8621b3-027c-497d-963b-06319650f048"; + private static final String sourceMultidiskDomainXml = + "\n" + + " i-2-3-VM\n" + + " 91860126-7dda-4876-ac1e-48d06cd4b2eb\n" + + " Apple Mac OS X 10.6 (32-bit)\n" + + " 524288\n" + + " 524288\n" + + " 1\n" + + " \n" + + " 250\n" + + " \n" + + " \n" + + " \n" + + " Apache Software Foundation\n" + + " CloudStack KVM Hypervisor\n" + + " 91860126-7dda-4876-ac1e-48d06cd4b2eb\n" + + " \n" + + " \n" + + " \n" + + " hvm\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " destroy\n" + + " restart\n" + + " destroy\n" + + " \n" + + " /usr/libexec/qemu-kvm\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " e8141f63b5364a7f8cbb\n" + + " \n" + + "
\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + "
\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " bf8621b3027c497d963b\n" + + " \n" + + "
\n" + + " \n" + + " \n" + + " \n" + + "
\n" + + " \n" + + " \n" + + " \n" + + "
\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + "
\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + "