diff --git a/.github/workflows/ui.yml b/.github/workflows/ui.yml index 56f757133b7..88d4a70e4c2 100644 --- a/.github/workflows/ui.yml +++ b/.github/workflows/ui.yml @@ -56,6 +56,7 @@ jobs: npm run test:unit - uses: codecov/codecov-action@v4 + if: github.repository == 'apache/cloudstack' with: working-directory: ui files: ./coverage/lcov.info diff --git a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java index e5593f10460..c781c07c227 100644 --- a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java +++ b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java @@ -823,7 +823,7 @@ public class AgentProperties{ private T defaultValue; private Class typeClass; - Property(String name, T value) { + public Property(String name, T value) { init(name, value); } diff --git a/api/src/main/java/com/cloud/host/Host.java b/api/src/main/java/com/cloud/host/Host.java index 56b4ed75a31..a3b6ccadc01 100644 --- a/api/src/main/java/com/cloud/host/Host.java +++ b/api/src/main/java/com/cloud/host/Host.java @@ -53,9 +53,12 @@ public interface Host extends StateObject, Identity, Partition, HAResour return strs; } } - public static final String HOST_UEFI_ENABLE = "host.uefi.enable"; - public static final String HOST_VOLUME_ENCRYPTION = "host.volume.encryption"; - public static final String HOST_INSTANCE_CONVERSION = "host.instance.conversion"; + + String HOST_UEFI_ENABLE = "host.uefi.enable"; + String HOST_VOLUME_ENCRYPTION = "host.volume.encryption"; + String HOST_INSTANCE_CONVERSION = "host.instance.conversion"; + String HOST_OVFTOOL_VERSION = "host.ovftool.version"; + String HOST_VIRTV2V_VERSION = "host.virtv2v.version"; /** * @return name of the machine. diff --git a/api/src/main/java/com/cloud/storage/VolumeApiService.java b/api/src/main/java/com/cloud/storage/VolumeApiService.java index bb69b5b6650..4182728c204 100644 --- a/api/src/main/java/com/cloud/storage/VolumeApiService.java +++ b/api/src/main/java/com/cloud/storage/VolumeApiService.java @@ -171,6 +171,13 @@ public interface VolumeApiService { * */ boolean doesStoragePoolSupportDiskOffering(StoragePool destPool, DiskOffering diskOffering); + + /** + * Checks if the storage pool supports the required disk offering tags + * destPool the storage pool to check the disk offering tags + * diskOfferingTags the tags that should be supported + * return whether the tags are supported in the storage pool + */ boolean doesStoragePoolSupportDiskOfferingTags(StoragePool destPool, String diskOfferingTags); Volume destroyVolume(long volumeId, Account caller, boolean expunge, boolean forceExpunge); diff --git a/api/src/main/java/com/cloud/user/AccountService.java b/api/src/main/java/com/cloud/user/AccountService.java index e2c3bed0c29..c0ebcf09f59 100644 --- a/api/src/main/java/com/cloud/user/AccountService.java +++ b/api/src/main/java/com/cloud/user/AccountService.java @@ -87,6 +87,8 @@ public interface AccountService { boolean isDomainAdmin(Long accountId); + boolean isResourceDomainAdmin(Long accountId); + boolean isNormalUser(long accountId); User getActiveUserByRegistrationToken(String registrationToken); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/acl/project/ListProjectRolesCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/acl/project/ListProjectRolesCmd.java index e876dbc9b58..dedbb410ea5 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/acl/project/ListProjectRolesCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/acl/project/ListProjectRolesCmd.java @@ -72,7 +72,7 @@ public class ListProjectRolesCmd extends BaseListCmd { @Override public void execute() { - List projectRoles; + List projectRoles = new ArrayList<>(); if (getProjectId() != null && getProjectRoleId() != null) { projectRoles = Collections.singletonList(projRoleService.findProjectRole(getProjectRoleId(), getProjectId())); } else if (StringUtils.isNotBlank(getRoleName())) { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/cluster/ListClustersCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/cluster/ListClustersCmd.java index 362913a1138..90ec9d1ff07 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/cluster/ListClustersCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/cluster/ListClustersCmd.java @@ -128,19 +128,19 @@ public class ListClustersCmd extends BaseListCmd { protected Pair, Integer> getClusterResponses() { Pair, Integer> result = _mgr.searchForClusters(this); - List clusterResponses = new ArrayList(); + List clusterResponses = new ArrayList<>(); for (Cluster cluster : result.first()) { ClusterResponse clusterResponse = _responseGenerator.createClusterResponse(cluster, showCapacities); clusterResponse.setObjectName("cluster"); clusterResponses.add(clusterResponse); } - return new Pair, Integer>(clusterResponses, result.second()); + return new Pair<>(clusterResponses, result.second()); } @Override public void execute() { Pair, Integer> clusterResponses = getClusterResponses(); - ListResponse response = new ListResponse(); + ListResponse response = new ListResponse<>(); response.setResponses(clusterResponses.first(), clusterResponses.second()); response.setResponseName(getCommandName()); this.setResponseObject(response); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/pod/ListPodsByCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/pod/ListPodsByCmd.java index 5ad0b457ced..10370b4c78a 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/pod/ListPodsByCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/pod/ListPodsByCmd.java @@ -86,8 +86,8 @@ public class ListPodsByCmd extends BaseListCmd { @Override public void execute() { Pair, Integer> result = _mgr.searchForPods(this); - ListResponse response = new ListResponse(); - List podResponses = new ArrayList(); + ListResponse response = new ListResponse<>(); + List podResponses = new ArrayList<>(); for (Pod pod : result.first()) { PodResponse podResponse = _responseGenerator.createPodResponse(pod, showCapacities); podResponse.setObjectName("pod"); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/resource/ListCapacityCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/resource/ListCapacityCmd.java index 6b31c4cc43c..e65326cd874 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/resource/ListCapacityCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/resource/ListCapacityCmd.java @@ -132,14 +132,16 @@ public class ListCapacityCmd extends BaseListCmd { Collections.sort(capacityResponses, new Comparator() { public int compare(CapacityResponse resp1, CapacityResponse resp2) { int res = resp1.getZoneName().compareTo(resp2.getZoneName()); + // Group by zone if (res != 0) { return res; - } else { - return resp1.getCapacityType().compareTo(resp2.getCapacityType()); } + // Sort by capacity type only if not already sorted by usage + return (getSortBy() != null) ? 0 : resp1.getCapacityType().compareTo(resp2.getCapacityType()); } }); + response.setResponses(capacityResponses); response.setResponseName(getCommandName()); this.setResponseObject(response); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/systemvm/ScaleSystemVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/systemvm/ScaleSystemVMCmd.java index 06e57674c53..d822d4fc26e 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/systemvm/ScaleSystemVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/systemvm/ScaleSystemVMCmd.java @@ -74,7 +74,7 @@ public class ScaleSystemVMCmd extends BaseAsyncCmd { } public Map getDetails() { - return details; + return convertDetailsToMap(details); } ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/systemvm/UpgradeSystemVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/systemvm/UpgradeSystemVMCmd.java index 5abe90e3f58..a42f0052249 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/systemvm/UpgradeSystemVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/systemvm/UpgradeSystemVMCmd.java @@ -68,7 +68,7 @@ public class UpgradeSystemVMCmd extends BaseCmd { } public Map getDetails() { - return details; + return convertDetailsToMap(details); } ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/config/ListCapabilitiesCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/config/ListCapabilitiesCmd.java index 0cecbb37020..bd3f39a09aa 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/config/ListCapabilitiesCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/config/ListCapabilitiesCmd.java @@ -72,6 +72,7 @@ public class ListCapabilitiesCmd extends BaseCmd { response.setInstancesDisksStatsRetentionTime((Integer) capabilities.get(ApiConstants.INSTANCES_DISKS_STATS_RETENTION_TIME)); response.setSharedFsVmMinCpuCount((Integer)capabilities.get(ApiConstants.SHAREDFSVM_MIN_CPU_COUNT)); response.setSharedFsVmMinRamSize((Integer)capabilities.get(ApiConstants.SHAREDFSVM_MIN_RAM_SIZE)); + response.setDynamicScalingEnabled((Boolean) capabilities.get(ApiConstants.DYNAMIC_SCALING_ENABLED)); response.setObjectName("capability"); response.setResponseName(getCommandName()); this.setResponseObject(response); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/template/CreateTemplateCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/template/CreateTemplateCmd.java index 0a7bf291843..5f09ac6698d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/template/CreateTemplateCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/template/CreateTemplateCmd.java @@ -20,6 +20,7 @@ import java.util.Collection; import java.util.List; import java.util.Map; +import com.cloud.cpu.CPU; import org.apache.cloudstack.acl.SecurityChecker; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiCommandResourceType; @@ -148,6 +149,11 @@ public class CreateTemplateCmd extends BaseAsyncCreateCmd implements UserCmd { since = "4.19.0") private String accountName; + @Parameter(name = ApiConstants.ARCH, type = CommandType.STRING, + description = "the CPU arch of the template. Valid options are: x86_64, aarch64. Defaults to x86_64", + since = "4.20.2") + private String arch; + // /////////////////////////////////////////////////// // ///////////////// Accessors /////////////////////// // /////////////////////////////////////////////////// @@ -234,6 +240,10 @@ public class CreateTemplateCmd extends BaseAsyncCreateCmd implements UserCmd { return accountName; } + public CPU.CPUArch getArch() { + return CPU.CPUArch.fromType(arch); + } + // /////////////////////////////////////////////////// // ///////////// API Implementation/////////////////// // /////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/RegisterUserDataCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/RegisterUserDataCmd.java index 41d865d678c..4588d734847 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/RegisterUserDataCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/RegisterUserDataCmd.java @@ -43,7 +43,7 @@ import com.cloud.network.NetworkModel; import com.cloud.user.UserData; @APICommand(name = "registerUserData", - description = "Register a new userdata.", + description = "Register a new User Data.", since = "4.18", responseObject = SuccessResponse.class, requestHasSensitiveInfo = false, @@ -56,33 +56,33 @@ public class RegisterUserDataCmd extends BaseCmd { //////////////// API parameters ///////////////////// ///////////////////////////////////////////////////// - @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, required = true, description = "Name of the userdata") + @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, required = true, description = "Name of the User Data") private String name; //Owner information - @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, description = "an optional account for the userdata. Must be used with domainId.") + @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, description = "an optional account for the User Data. Must be used with domainId.") private String accountName; @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, entityType = DomainResponse.class, - description = "an optional domainId for the userdata. If the account parameter is used, domainId must also be used.") + description = "an optional domainId for the User Data. If the account parameter is used, domainId must also be used.") private Long domainId; - @Parameter(name = ApiConstants.PROJECT_ID, type = CommandType.UUID, entityType = ProjectResponse.class, description = "an optional project for the userdata") + @Parameter(name = ApiConstants.PROJECT_ID, type = CommandType.UUID, entityType = ProjectResponse.class, description = "an optional project for the User Data") private Long projectId; @Parameter(name = ApiConstants.USER_DATA, type = CommandType.STRING, required = true, - description = "Base64 encoded userdata content. " + + description = "Base64 encoded User Data content. " + "Using HTTP GET (via querystring), you can send up to 4KB of data after base64 encoding. " + - "Using HTTP POST (via POST body), you can send up to 1MB of data after base64 encoding. " + - "You also need to change vm.userdata.max.length value", + "Using HTTP POST (via POST body), you can send up to 32KB of data after base64 encoding, " + + "which can be increased upto 1MB using the vm.userdata.max.length setting", length = 1048576) private String userData; - @Parameter(name = ApiConstants.PARAMS, type = CommandType.STRING, description = "comma separated list of variables declared in userdata content") + @Parameter(name = ApiConstants.PARAMS, type = CommandType.STRING, description = "comma separated list of variables declared in the User Data content") private String params; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DestroyVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DestroyVMCmd.java index aa121162cb4..18a9d2058a6 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DestroyVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DestroyVMCmd.java @@ -140,7 +140,8 @@ public class DestroyVMCmd extends BaseAsyncCmd implements UserCmd { if (responses != null && !responses.isEmpty()) { response = responses.get(0); } - response.setResponseName("virtualmachine"); + response.setResponseName(getCommandName()); + response.setObjectName("virtualmachine"); setResponseObject(response); } else { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to destroy vm"); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/zone/ListZonesCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/zone/ListZonesCmd.java index d926257437e..8d371bb6761 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/zone/ListZonesCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/zone/ListZonesCmd.java @@ -34,8 +34,6 @@ import org.apache.cloudstack.api.response.ZoneResponse; requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) public class ListZonesCmd extends BaseListCmd implements UserCmd { - private static final String s_name = "listzonesresponse"; - ///////////////////////////////////////////////////// //////////////// API parameters ///////////////////// ///////////////////////////////////////////////////// @@ -113,11 +111,6 @@ public class ListZonesCmd extends BaseListCmd implements UserCmd { /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// - @Override - public String getCommandName() { - return s_name; - } - @Override public void execute() { ListResponse response = _queryService.listDataCenters(this); diff --git a/api/src/main/java/org/apache/cloudstack/api/response/CapabilitiesResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/CapabilitiesResponse.java index 3861ac455ed..ff2e33b1389 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/CapabilitiesResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/CapabilitiesResponse.java @@ -136,6 +136,10 @@ public class CapabilitiesResponse extends BaseResponse { @Param(description = "the min Ram size for the service offering used by the shared filesystem instance", since = "4.20.0") private Integer sharedFsVmMinRamSize; + @SerializedName(ApiConstants.DYNAMIC_SCALING_ENABLED) + @Param(description = "true if dynamically scaling for instances is enabled", since = "4.21.0") + private Boolean dynamicScalingEnabled; + public void setSecurityGroupsEnabled(boolean securityGroupsEnabled) { this.securityGroupsEnabled = securityGroupsEnabled; } @@ -247,4 +251,8 @@ public class CapabilitiesResponse extends BaseResponse { public void setSharedFsVmMinRamSize(Integer sharedFsVmMinRamSize) { this.sharedFsVmMinRamSize = sharedFsVmMinRamSize; } + + public void setDynamicScalingEnabled(Boolean dynamicScalingEnabled) { + this.dynamicScalingEnabled = dynamicScalingEnabled; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/StatsResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/StatsResponse.java index 5dd76fa5eef..287d78bb612 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/StatsResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/StatsResponse.java @@ -27,7 +27,7 @@ import com.google.gson.annotations.SerializedName; public class StatsResponse extends BaseResponse { @SerializedName("timestamp") - @Param(description = "the time when the VM stats were collected. The format is \"yyyy-MM-dd hh:mm:ss\"") + @Param(description = "the time when the VM stats were collected. The format is 'yyyy-MM-dd hh:mm:ss'") private Date timestamp; @SerializedName("cpuused") diff --git a/api/src/main/java/org/apache/cloudstack/api/response/UserDataResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/UserDataResponse.java index 2dfc66fa7d5..ce344596aeb 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/UserDataResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/UserDataResponse.java @@ -27,41 +27,41 @@ import org.apache.cloudstack.api.EntityReference; public class UserDataResponse extends BaseResponseWithAnnotations implements ControlledEntityResponse { @SerializedName(ApiConstants.ID) - @Param(description = "ID of the ssh keypair") + @Param(description = "ID of the User Data") private String id; @SerializedName(ApiConstants.NAME) - @Param(description = "Name of the userdata") + @Param(description = "Name of the User Data") private String name; - @SerializedName(ApiConstants.ACCOUNT_ID) @Param(description="the owner id of the userdata") + @SerializedName(ApiConstants.ACCOUNT_ID) @Param(description="the owner id of the User Data") private String accountId; - @SerializedName(ApiConstants.ACCOUNT) @Param(description="the owner of the userdata") + @SerializedName(ApiConstants.ACCOUNT) @Param(description="the owner of the User Data") private String accountName; @SerializedName(ApiConstants.PROJECT_ID) - @Param(description = "the project id of the userdata", since = "4.19.1") + @Param(description = "the project id of the User Data", since = "4.19.1") private String projectId; @SerializedName(ApiConstants.PROJECT) - @Param(description = "the project name of the userdata", since = "4.19.1") + @Param(description = "the project name of the User Data", since = "4.19.1") private String projectName; - @SerializedName(ApiConstants.DOMAIN_ID) @Param(description="the domain id of the userdata owner") + @SerializedName(ApiConstants.DOMAIN_ID) @Param(description="the domain id of the User Data owner") private String domainId; - @SerializedName(ApiConstants.DOMAIN) @Param(description="the domain name of the userdata owner") + @SerializedName(ApiConstants.DOMAIN) @Param(description="the domain name of the User Data owner") private String domain; @SerializedName(ApiConstants.DOMAIN_PATH) - @Param(description = "path of the domain to which the userdata owner belongs", since = "4.19.2.0") + @Param(description = "path of the domain to which the User Data owner belongs", since = "4.19.2.0") private String domainPath; - @SerializedName(ApiConstants.USER_DATA) @Param(description="base64 encoded userdata content") + @SerializedName(ApiConstants.USER_DATA) @Param(description="base64 encoded User Data content") private String userData; - @SerializedName(ApiConstants.PARAMS) @Param(description="list of parameters which contains the list of keys or string parameters that are needed to be passed for any variables declared in userdata") + @SerializedName(ApiConstants.PARAMS) @Param(description="list of parameters which contains the list of keys or string parameters that are needed to be passed for any variables declared in the User Data") private String params; public UserDataResponse() { diff --git a/api/src/main/java/org/apache/cloudstack/query/QueryService.java b/api/src/main/java/org/apache/cloudstack/query/QueryService.java index 0a5721abdc1..0f658e5d20d 100644 --- a/api/src/main/java/org/apache/cloudstack/query/QueryService.java +++ b/api/src/main/java/org/apache/cloudstack/query/QueryService.java @@ -112,11 +112,11 @@ public interface QueryService { ConfigKey AllowUserViewDestroyedVM = new ConfigKey<>("Advanced", Boolean.class, "allow.user.view.destroyed.vm", "false", "Determines whether users can view their destroyed or expunging vm ", true, ConfigKey.Scope.Account); - static final ConfigKey UserVMDeniedDetails = new ConfigKey<>(String.class, + ConfigKey UserVMDeniedDetails = new ConfigKey<>(String.class, "user.vm.denied.details", "Advanced", "rootdisksize, cpuOvercommitRatio, memoryOvercommitRatio, Message.ReservedCapacityFreed.Flag", "Determines whether users can view certain VM settings. When set to empty, default value used is: rootdisksize, cpuOvercommitRatio, memoryOvercommitRatio, Message.ReservedCapacityFreed.Flag.", true, ConfigKey.Scope.Global, null, null, null, null, null, ConfigKey.Kind.CSV, null); - static final ConfigKey UserVMReadOnlyDetails = new ConfigKey<>(String.class, + ConfigKey UserVMReadOnlyDetails = new ConfigKey<>(String.class, "user.vm.readonly.details", "Advanced", "dataDiskController, rootDiskController", "List of read-only VM settings/details as comma separated string", true, ConfigKey.Scope.Global, null, null, null, null, null, ConfigKey.Kind.CSV, null); @@ -125,16 +125,20 @@ public interface QueryService { "network offering, zones), we use the flag to determine if the entities should be sorted ascending (when flag is true) " + "or descending (when flag is false). Within the scope of the config all users see the same result.", true, ConfigKey.Scope.Global); - public static final ConfigKey AllowUserViewAllDomainAccounts = new ConfigKey<>("Advanced", Boolean.class, + ConfigKey AllowUserViewAllDomainAccounts = new ConfigKey<>("Advanced", Boolean.class, "allow.user.view.all.domain.accounts", "false", "Determines whether users can view all user accounts within the same domain", true, ConfigKey.Scope.Domain); - static final ConfigKey SharePublicTemplatesWithOtherDomains = new ConfigKey<>("Advanced", Boolean.class, "share.public.templates.with.other.domains", "true", + ConfigKey AllowUserViewAllDataCenters = new ConfigKey<>("Advanced", Boolean.class, "allow.user.view.all.zones", "true", + "Determines whether for instance a Resource Admin can view zones that are not dedicated to them.", true, ConfigKey.Scope.Domain); + + ConfigKey SharePublicTemplatesWithOtherDomains = new ConfigKey<>("Advanced", Boolean.class, "share.public.templates.with.other.domains", "true", "If false, templates of this domain will not show up in the list templates of other domains.", true, ConfigKey.Scope.Domain); ConfigKey ReturnVmStatsOnVmList = new ConfigKey<>("Advanced", Boolean.class, "list.vm.default.details.stats", "true", "Determines whether VM stats should be returned when details are not explicitly specified in listVirtualMachines API request. When false, details default to [group, nics, secgrp, tmpl, servoff, diskoff, backoff, iso, volume, min, affgrp]. When true, all details are returned including 'stats'.", true, ConfigKey.Scope.Global); + ListResponse searchForUsers(ResponseObject.ResponseView responseView, ListUsersCmd cmd) throws PermissionDeniedException; ListResponse searchForUsers(Long domainId, boolean recursive) throws PermissionDeniedException; diff --git a/core/src/main/java/com/cloud/agent/api/CheckVolumeAnswer.java b/core/src/main/java/com/cloud/agent/api/CheckVolumeAnswer.java index 5a32ab59a7a..07b7e102df9 100644 --- a/core/src/main/java/com/cloud/agent/api/CheckVolumeAnswer.java +++ b/core/src/main/java/com/cloud/agent/api/CheckVolumeAnswer.java @@ -17,22 +17,33 @@ package com.cloud.agent.api; +import org.apache.cloudstack.storage.volume.VolumeOnStorageTO; + +import java.util.Map; + public class CheckVolumeAnswer extends Answer { private long size; + private Map volumeDetails; CheckVolumeAnswer() { } - public CheckVolumeAnswer(CheckVolumeCommand cmd, String details, long size) { - super(cmd, true, details); + public CheckVolumeAnswer(CheckVolumeCommand cmd, final boolean success, String details, long size, + Map volumeDetails) { + super(cmd, success, details); this.size = size; + this.volumeDetails = volumeDetails; } public long getSize() { return size; } + public Map getVolumeDetails() { + return volumeDetails; + } + public String getString() { return "CheckVolumeAnswer [size=" + size + "]"; } diff --git a/core/src/main/java/com/cloud/agent/api/ConvertInstanceAnswer.java b/core/src/main/java/com/cloud/agent/api/ConvertInstanceAnswer.java index 174348f4f18..8092ab9b43f 100644 --- a/core/src/main/java/com/cloud/agent/api/ConvertInstanceAnswer.java +++ b/core/src/main/java/com/cloud/agent/api/ConvertInstanceAnswer.java @@ -16,8 +16,6 @@ // under the License. package com.cloud.agent.api; -import org.apache.cloudstack.vm.UnmanagedInstanceTO; - public class ConvertInstanceAnswer extends Answer { private String temporaryConvertUuid; @@ -25,16 +23,6 @@ public class ConvertInstanceAnswer extends Answer { public ConvertInstanceAnswer() { super(); } - private UnmanagedInstanceTO convertedInstance; - - public ConvertInstanceAnswer(Command command, boolean success, String details) { - super(command, success, details); - } - - public ConvertInstanceAnswer(Command command, UnmanagedInstanceTO convertedInstance) { - super(command, true, ""); - this.convertedInstance = convertedInstance; - } public ConvertInstanceAnswer(Command command, String temporaryConvertUuid) { super(command, true, ""); @@ -44,8 +32,4 @@ public class ConvertInstanceAnswer extends Answer { public String getTemporaryConvertUuid() { return temporaryConvertUuid; } - - public UnmanagedInstanceTO getConvertedInstance() { - return convertedInstance; - } } diff --git a/core/src/main/java/com/cloud/agent/api/ConvertInstanceCommand.java b/core/src/main/java/com/cloud/agent/api/ConvertInstanceCommand.java index b8250903f85..f938d0ac55d 100644 --- a/core/src/main/java/com/cloud/agent/api/ConvertInstanceCommand.java +++ b/core/src/main/java/com/cloud/agent/api/ConvertInstanceCommand.java @@ -20,13 +20,10 @@ import com.cloud.agent.api.to.DataStoreTO; import com.cloud.agent.api.to.RemoteInstanceTO; import com.cloud.hypervisor.Hypervisor; -import java.util.List; - public class ConvertInstanceCommand extends Command { private RemoteInstanceTO sourceInstance; private Hypervisor.HypervisorType destinationHypervisorType; - private List destinationStoragePools; private DataStoreTO conversionTemporaryLocation; private String templateDirOnConversionLocation; private boolean checkConversionSupport; @@ -36,12 +33,10 @@ public class ConvertInstanceCommand extends Command { public ConvertInstanceCommand() { } - public ConvertInstanceCommand(RemoteInstanceTO sourceInstance, Hypervisor.HypervisorType destinationHypervisorType, - List destinationStoragePools, DataStoreTO conversionTemporaryLocation, + public ConvertInstanceCommand(RemoteInstanceTO sourceInstance, Hypervisor.HypervisorType destinationHypervisorType, DataStoreTO conversionTemporaryLocation, String templateDirOnConversionLocation, boolean checkConversionSupport, boolean exportOvfToConversionLocation) { this.sourceInstance = sourceInstance; this.destinationHypervisorType = destinationHypervisorType; - this.destinationStoragePools = destinationStoragePools; this.conversionTemporaryLocation = conversionTemporaryLocation; this.templateDirOnConversionLocation = templateDirOnConversionLocation; this.checkConversionSupport = checkConversionSupport; @@ -56,10 +51,6 @@ public class ConvertInstanceCommand extends Command { return destinationHypervisorType; } - public List getDestinationStoragePools() { - return destinationStoragePools; - } - public DataStoreTO getConversionTemporaryLocation() { return conversionTemporaryLocation; } diff --git a/core/src/main/java/com/cloud/agent/api/CopyRemoteVolumeAnswer.java b/core/src/main/java/com/cloud/agent/api/CopyRemoteVolumeAnswer.java index e79005be71b..4aec0b26581 100644 --- a/core/src/main/java/com/cloud/agent/api/CopyRemoteVolumeAnswer.java +++ b/core/src/main/java/com/cloud/agent/api/CopyRemoteVolumeAnswer.java @@ -17,21 +17,28 @@ package com.cloud.agent.api; +import org.apache.cloudstack.storage.volume.VolumeOnStorageTO; + +import java.util.Map; + public class CopyRemoteVolumeAnswer extends Answer { private String remoteIp; private String filename; private long size; + private Map volumeDetails; CopyRemoteVolumeAnswer() { } - public CopyRemoteVolumeAnswer(CopyRemoteVolumeCommand cmd, String details, String filename, long size) { - super(cmd, true, details); + public CopyRemoteVolumeAnswer(CopyRemoteVolumeCommand cmd, final boolean success, String details, String filename, long size, + Map volumeDetails) { + super(cmd, success, details); this.remoteIp = cmd.getRemoteIp(); this.filename = filename; this.size = size; + this.volumeDetails = volumeDetails; } public String getRemoteIp() { @@ -54,6 +61,10 @@ public class CopyRemoteVolumeAnswer extends Answer { return size; } + public Map getVolumeDetails() { + return volumeDetails; + } + public String getString() { return "CopyRemoteVolumeAnswer [remoteIp=" + remoteIp + "]"; } diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java index b8a25a11b5c..338eb773257 100644 --- a/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java +++ b/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java @@ -39,9 +39,7 @@ import java.util.Map; import javax.net.ssl.HttpsURLConnection; import javax.net.ssl.SSLContext; -import javax.net.ssl.TrustManager; -import org.apache.cloudstack.utils.security.SSLUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.httpclient.HttpStatus; import org.apache.commons.io.IOUtils; @@ -55,6 +53,7 @@ import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; +import org.apache.http.ssl.SSLContexts; import org.apache.http.util.EntityUtils; import com.cloud.utils.Pair; @@ -120,10 +119,10 @@ public class HttpsDirectTemplateDownloader extends DirectTemplateDownloaderImpl String password = "changeit"; defaultKeystore.load(is, password.toCharArray()); } - TrustManager[] tm = HttpsMultiTrustManager.getTrustManagersFromKeyStores(customKeystore, defaultKeystore); - SSLContext sslContext = SSLUtils.getSSLContext(); - sslContext.init(null, tm, null); - return sslContext; + return SSLContexts.custom() + .loadTrustMaterial(customKeystore, null) + .loadTrustMaterial(defaultKeystore, null) + .build(); } catch (KeyStoreException | NoSuchAlgorithmException | CertificateException | IOException | KeyManagementException e) { logger.error(String.format("Failure getting SSL context for HTTPS downloader, using default SSL context: %s", e.getMessage()), e); try { diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/HttpsMultiTrustManager.java b/core/src/main/java/org/apache/cloudstack/direct/download/HttpsMultiTrustManager.java deleted file mode 100644 index fe47847c36c..00000000000 --- a/core/src/main/java/org/apache/cloudstack/direct/download/HttpsMultiTrustManager.java +++ /dev/null @@ -1,102 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -package org.apache.cloudstack.direct.download; - -import java.security.KeyStore; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.security.cert.CertificateException; -import java.security.cert.X509Certificate; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import javax.net.ssl.TrustManager; -import javax.net.ssl.TrustManagerFactory; -import javax.net.ssl.X509TrustManager; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterables; - -public class HttpsMultiTrustManager implements X509TrustManager { - - private final List trustManagers; - - public HttpsMultiTrustManager(KeyStore... keystores) { - List trustManagers = new ArrayList<>(); - trustManagers.add(getTrustManager(null)); - for (KeyStore keystore : keystores) { - trustManagers.add(getTrustManager(keystore)); - } - this.trustManagers = ImmutableList.copyOf(trustManagers); - } - - public static TrustManager[] getTrustManagersFromKeyStores(KeyStore... keyStore) { - return new TrustManager[] { new HttpsMultiTrustManager(keyStore) }; - - } - - @Override - public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException { - for (X509TrustManager trustManager : trustManagers) { - try { - trustManager.checkClientTrusted(chain, authType); - return; - } catch (CertificateException ignored) {} - } - throw new CertificateException("None of the TrustManagers trust this certificate chain"); - } - - @Override - public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException { - for (X509TrustManager trustManager : trustManagers) { - try { - trustManager.checkServerTrusted(chain, authType); - return; - } catch (CertificateException ignored) {} - } - throw new CertificateException("None of the TrustManagers trust this certificate chain"); - } - - @Override - public X509Certificate[] getAcceptedIssuers() { - ImmutableList.Builder certificates = ImmutableList.builder(); - for (X509TrustManager trustManager : trustManagers) { - for (X509Certificate cert : trustManager.getAcceptedIssuers()) { - certificates.add(cert); - } - } - return Iterables.toArray(certificates.build(), X509Certificate.class); - } - - public X509TrustManager getTrustManager(KeyStore keystore) { - return getTrustManager(TrustManagerFactory.getDefaultAlgorithm(), keystore); - } - - public X509TrustManager getTrustManager(String algorithm, KeyStore keystore) { - TrustManagerFactory factory; - try { - factory = TrustManagerFactory.getInstance(algorithm); - factory.init(keystore); - return Iterables.getFirst(Iterables.filter( - Arrays.asList(factory.getTrustManagers()), X509TrustManager.class), null); - } catch (NoSuchAlgorithmException | KeyStoreException e) { - e.printStackTrace(); - } - return null; - } -} diff --git a/engine/api/src/main/java/com/cloud/vm/VirtualMachineGuru.java b/engine/api/src/main/java/com/cloud/vm/VirtualMachineGuru.java index f8032bf4b0e..76f0830f369 100644 --- a/engine/api/src/main/java/com/cloud/vm/VirtualMachineGuru.java +++ b/engine/api/src/main/java/com/cloud/vm/VirtualMachineGuru.java @@ -24,6 +24,7 @@ import com.cloud.utils.PasswordGenerator; import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.ca.CAManager; import org.apache.cloudstack.framework.ca.Certificate; +import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.utils.security.CertUtils; import org.apache.cloudstack.utils.security.KeyStoreUtils; @@ -37,6 +38,9 @@ import java.util.Base64; */ public interface VirtualMachineGuru { + static final ConfigKey NTPServerConfig = new ConfigKey(String.class, "ntp.server.list", "Advanced", null, + "Comma separated list of NTP servers to configure in System VMs", true, ConfigKey.Scope.Global, null, null, null, null, null, ConfigKey.Kind.CSV, null); + boolean finalizeVirtualMachineProfile(VirtualMachineProfile profile, DeployDestination dest, ReservationContext context); /** diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java index 9137d1c1dff..aff528efede 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java @@ -54,6 +54,7 @@ import org.apache.cloudstack.utils.identity.ManagementServerNode; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.ThreadContext; @@ -703,11 +704,25 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl Map detailsMap = readyAnswer.getDetailsMap(); if (detailsMap != null) { String uefiEnabled = detailsMap.get(Host.HOST_UEFI_ENABLE); + String virtv2vVersion = detailsMap.get(Host.HOST_VIRTV2V_VERSION); + String ovftoolVersion = detailsMap.get(Host.HOST_OVFTOOL_VERSION); logger.debug("Got HOST_UEFI_ENABLE [{}] for host [{}]:", uefiEnabled, host); - if (uefiEnabled != null) { + if (ObjectUtils.anyNotNull(uefiEnabled, virtv2vVersion, ovftoolVersion)) { _hostDao.loadDetails(host); + boolean updateNeeded = false; if (!uefiEnabled.equals(host.getDetails().get(Host.HOST_UEFI_ENABLE))) { host.getDetails().put(Host.HOST_UEFI_ENABLE, uefiEnabled); + updateNeeded = true; + } + if (StringUtils.isNotBlank(virtv2vVersion) && !virtv2vVersion.equals(host.getDetails().get(Host.HOST_VIRTV2V_VERSION))) { + host.getDetails().put(Host.HOST_VIRTV2V_VERSION, virtv2vVersion); + updateNeeded = true; + } + if (StringUtils.isNotBlank(ovftoolVersion) && !ovftoolVersion.equals(host.getDetails().get(Host.HOST_OVFTOOL_VERSION))) { + host.getDetails().put(Host.HOST_OVFTOOL_VERSION, ovftoolVersion); + updateNeeded = true; + } + if (updateNeeded) { _hostDao.saveDetails(host); } } diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java index e758d0b6832..a9c15afc2cf 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java @@ -21,6 +21,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Comparator; +import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; @@ -1292,6 +1293,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra IPAddressVO lockedIpVO = _ipAddressDao.acquireInLockTable(ipVO.getId()); validateLockedRequestedIp(ipVO, lockedIpVO); lockedIpVO.setState(IPAddressVO.State.Allocated); + lockedIpVO.setAllocatedTime(new Date()); _ipAddressDao.update(lockedIpVO.getId(), lockedIpVO); } finally { _ipAddressDao.releaseFromLockTable(ipVO.getId()); diff --git a/engine/schema/src/main/java/com/cloud/network/as/dao/AutoScaleVmGroupVmMapDao.java b/engine/schema/src/main/java/com/cloud/network/as/dao/AutoScaleVmGroupVmMapDao.java index 718511746c2..9775f8ad5b1 100644 --- a/engine/schema/src/main/java/com/cloud/network/as/dao/AutoScaleVmGroupVmMapDao.java +++ b/engine/schema/src/main/java/com/cloud/network/as/dao/AutoScaleVmGroupVmMapDao.java @@ -37,4 +37,6 @@ public interface AutoScaleVmGroupVmMapDao extends GenericDao vmIds, Long batchSize); + + int getErroredInstanceCount(long vmGroupId); } diff --git a/engine/schema/src/main/java/com/cloud/network/as/dao/AutoScaleVmGroupVmMapDaoImpl.java b/engine/schema/src/main/java/com/cloud/network/as/dao/AutoScaleVmGroupVmMapDaoImpl.java index 1ae55d97da2..b2f4e578a82 100644 --- a/engine/schema/src/main/java/com/cloud/network/as/dao/AutoScaleVmGroupVmMapDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/network/as/dao/AutoScaleVmGroupVmMapDaoImpl.java @@ -127,4 +127,13 @@ public class AutoScaleVmGroupVmMapDaoImpl extends GenericDaoBase sc = CountBy.create(); + sc.setParameters("vmGroupId", vmGroupId); + sc.setJoinParameters("vmSearch", "states", State.Error); + final List results = customSearch(sc, null); + return results.get(0); + } } diff --git a/engine/schema/src/main/java/com/cloud/network/security/SecurityGroupVMMapVO.java b/engine/schema/src/main/java/com/cloud/network/security/SecurityGroupVMMapVO.java index d12b9f9443f..59699cba1d4 100644 --- a/engine/schema/src/main/java/com/cloud/network/security/SecurityGroupVMMapVO.java +++ b/engine/schema/src/main/java/com/cloud/network/security/SecurityGroupVMMapVO.java @@ -50,6 +50,9 @@ public class SecurityGroupVMMapVO implements InternalIdentity { @Column(name = "ip4_address", table = "nics", insertable = false, updatable = false) private String guestIpAddress; + @Column(name = "ip6_address", table = "nics", insertable = false, updatable = false) + private String guestIpv6Address; + @Column(name = "state", table = "vm_instance", insertable = false, updatable = false) private State vmState; @@ -77,6 +80,10 @@ public class SecurityGroupVMMapVO implements InternalIdentity { return guestIpAddress; } + public String getGuestIpv6Address() { + return guestIpv6Address; + } + public long getInstanceId() { return instanceId; } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java index 0b40366a866..c751f81f927 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java @@ -72,7 +72,7 @@ public interface VMTemplateDao extends GenericDao, StateDao< VMTemplateVO findSystemVMTemplate(long zoneId); - VMTemplateVO findSystemVMReadyTemplate(long zoneId, HypervisorType hypervisorType); + VMTemplateVO findSystemVMReadyTemplate(long zoneId, HypervisorType hypervisorType, String preferredArch); List findSystemVMReadyTemplates(long zoneId, HypervisorType hypervisorType, String preferredArch); diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java index 12c00a3209a..b6796cf8f9d 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java @@ -23,6 +23,7 @@ import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import javax.inject.Inject; import javax.naming.ConfigurationException; @@ -578,11 +579,19 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem } @Override - public VMTemplateVO findSystemVMReadyTemplate(long zoneId, HypervisorType hypervisorType) { + public VMTemplateVO findSystemVMReadyTemplate(long zoneId, HypervisorType hypervisorType, String preferredArch) { List templates = listAllReadySystemVMTemplates(zoneId); if (CollectionUtils.isEmpty(templates)) { return null; } + if (StringUtils.isNotBlank(preferredArch)) { + // Sort the templates by preferred architecture first + templates = templates.stream() + .sorted(Comparator.comparing( + x -> !x.getArch().getType().equalsIgnoreCase(preferredArch) + )) + .collect(Collectors.toList()); + } if (hypervisorType == HypervisorType.Any) { return templates.get(0); } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java index e711c564015..26b033c8d79 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java @@ -64,11 +64,14 @@ import com.cloud.dc.dao.DataCenterDao; import com.cloud.dc.dao.DataCenterDaoImpl; import com.cloud.hypervisor.Hypervisor; import com.cloud.storage.DataStoreRole; +import com.cloud.storage.GuestOSVO; import com.cloud.storage.Storage; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.VMTemplateStorageResourceAssoc; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.VMTemplateZoneVO; +import com.cloud.storage.dao.GuestOSDao; +import com.cloud.storage.dao.GuestOSDaoImpl; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VMTemplateDaoImpl; import com.cloud.storage.dao.VMTemplateZoneDao; @@ -102,15 +105,13 @@ public class SystemVmTemplateRegistration { private static final String PARTIAL_TEMPLATE_FOLDER = String.format("/template/tmpl/%d/", Account.ACCOUNT_ID_SYSTEM); private static final String storageScriptsDir = "scripts/storage/secondary"; private static final Integer OTHER_LINUX_ID = 99; - private static final Integer LINUX_5_ID = 15; - private static final Integer LINUX_7_ID = 183; + private static Integer LINUX_12_ID = 363; private static final Integer SCRIPT_TIMEOUT = 1800000; private static final Integer LOCK_WAIT_TIMEOUT = 1200; protected static final List DOWNLOADABLE_TEMPLATE_ARCH_TYPES = Arrays.asList( CPU.CPUArch.arm64 ); - public static String CS_MAJOR_VERSION = null; public static String CS_TINY_VERSION = null; @@ -132,6 +133,8 @@ public class SystemVmTemplateRegistration { ClusterDao clusterDao; @Inject ConfigurationDao configurationDao; + @Inject + private GuestOSDao guestOSDao; private String systemVmTemplateVersion; @@ -147,6 +150,7 @@ public class SystemVmTemplateRegistration { imageStoreDetailsDao = new ImageStoreDetailsDaoImpl(); clusterDao = new ClusterDaoImpl(); configurationDao = new ConfigurationDaoImpl(); + guestOSDao = new GuestOSDaoImpl(); tempDownloadDir = new File(System.getProperty("java.io.tmpdir")); } @@ -320,7 +324,7 @@ public class SystemVmTemplateRegistration { public static final Map NewTemplateMap = new HashMap<>(); - public static final Map RouterTemplateConfigurationNames = new HashMap() { + public static final Map RouterTemplateConfigurationNames = new HashMap<>() { { put(Hypervisor.HypervisorType.KVM, "router.template.kvm"); put(Hypervisor.HypervisorType.VMware, "router.template.vmware"); @@ -331,14 +335,14 @@ public class SystemVmTemplateRegistration { } }; - public static final Map hypervisorGuestOsMap = new HashMap() { + public static Map hypervisorGuestOsMap = new HashMap<>() { { - put(Hypervisor.HypervisorType.KVM, LINUX_5_ID); + put(Hypervisor.HypervisorType.KVM, LINUX_12_ID); put(Hypervisor.HypervisorType.XenServer, OTHER_LINUX_ID); put(Hypervisor.HypervisorType.VMware, OTHER_LINUX_ID); - put(Hypervisor.HypervisorType.Hyperv, LINUX_5_ID); - put(Hypervisor.HypervisorType.LXC, LINUX_5_ID); - put(Hypervisor.HypervisorType.Ovm3, LINUX_7_ID); + put(Hypervisor.HypervisorType.Hyperv, LINUX_12_ID); + put(Hypervisor.HypervisorType.LXC, LINUX_12_ID); + put(Hypervisor.HypervisorType.Ovm3, LINUX_12_ID); } }; @@ -595,6 +599,23 @@ public class SystemVmTemplateRegistration { vmInstanceDao.updateSystemVmTemplateId(templateId, hypervisorType); } + private void updateSystemVmTemplateGuestOsId() { + String systemVmGuestOsName = "Debian GNU/Linux 12 (64-bit)"; // default + try { + GuestOSVO guestOS = guestOSDao.findOneByDisplayName(systemVmGuestOsName); + if (guestOS != null) { + LOGGER.debug("Updating SystemVM Template Guest OS [{}] id", systemVmGuestOsName); + SystemVmTemplateRegistration.LINUX_12_ID = Math.toIntExact(guestOS.getId()); + hypervisorGuestOsMap.put(Hypervisor.HypervisorType.KVM, LINUX_12_ID); + hypervisorGuestOsMap.put(Hypervisor.HypervisorType.Hyperv, LINUX_12_ID); + hypervisorGuestOsMap.put(Hypervisor.HypervisorType.LXC, LINUX_12_ID); + hypervisorGuestOsMap.put(Hypervisor.HypervisorType.Ovm3, LINUX_12_ID); + } + } catch (Exception e) { + LOGGER.warn("Couldn't update SystemVM Template Guest OS id, due to {}", e.getMessage()); + } + } + public void updateConfigurationParams(Map configParams) { for (Map.Entry config : configParams.entrySet()) { boolean updated = configurationDao.update(config.getKey(), config.getValue()); @@ -813,7 +834,8 @@ public class SystemVmTemplateRegistration { section.get("filename"), section.get("downloadurl"), section.get("checksum"), - hypervisorType.second())); + hypervisorType.second(), + section.get("guestos"))); } Ini.Section defaultSection = ini.get("default"); return defaultSection.get("version").trim(); @@ -965,6 +987,10 @@ public class SystemVmTemplateRegistration { private void updateRegisteredTemplateDetails(Long templateId, MetadataTemplateDetails templateDetails) { VMTemplateVO templateVO = vmTemplateDao.findById(templateId); templateVO.setTemplateType(Storage.TemplateType.SYSTEM); + GuestOSVO guestOS = guestOSDao.findOneByDisplayName(templateDetails.getGuestOs()); + if (guestOS != null) { + templateVO.setGuestOSId(guestOS.getId()); + } boolean updated = vmTemplateDao.update(templateVO.getId(), templateVO); if (!updated) { String errMsg = String.format("updateSystemVmTemplates:Exception while updating template with id %s to be marked as 'system'", templateId); @@ -980,9 +1006,13 @@ public class SystemVmTemplateRegistration { updateConfigurationParams(configParams); } - private void updateTemplateUrlAndChecksum(VMTemplateVO templateVO, MetadataTemplateDetails templateDetails) { + private void updateTemplateUrlChecksumAndGuestOsId(VMTemplateVO templateVO, MetadataTemplateDetails templateDetails) { templateVO.setUrl(templateDetails.getUrl()); templateVO.setChecksum(templateDetails.getChecksum()); + GuestOSVO guestOS = guestOSDao.findOneByDisplayName(templateDetails.getGuestOs()); + if (guestOS != null) { + templateVO.setGuestOSId(guestOS.getId()); + } boolean updated = vmTemplateDao.update(templateVO.getId(), templateVO); if (!updated) { String errMsg = String.format("updateSystemVmTemplates:Exception while updating 'url' and 'checksum' for hypervisor type %s", templateDetails.getHypervisorType()); @@ -1020,7 +1050,7 @@ public class SystemVmTemplateRegistration { VMTemplateVO templateVO = vmTemplateDao.findLatestTemplateByTypeAndHypervisorAndArch( templateDetails.getHypervisorType(), templateDetails.getArch(), Storage.TemplateType.SYSTEM); if (templateVO != null) { - updateTemplateUrlAndChecksum(templateVO, templateDetails); + updateTemplateUrlChecksumAndGuestOsId(templateVO, templateDetails); } } } @@ -1029,6 +1059,7 @@ public class SystemVmTemplateRegistration { public void updateSystemVmTemplates(final Connection conn) { LOGGER.debug("Updating System Vm template IDs"); + updateSystemVmTemplateGuestOsId(); Transaction.execute(new TransactionCallbackNoReturn() { @Override public void doInTransactionWithoutResult(final TransactionStatus status) { @@ -1076,15 +1107,17 @@ public class SystemVmTemplateRegistration { private final String checksum; private final CPU.CPUArch arch; private String downloadedFilePath; + private final String guestOs; MetadataTemplateDetails(Hypervisor.HypervisorType hypervisorType, String name, String filename, String url, - String checksum, CPU.CPUArch arch) { + String checksum, CPU.CPUArch arch, String guestOs) { this.hypervisorType = hypervisorType; this.name = name; this.filename = filename; this.url = url; this.checksum = checksum; this.arch = arch; + this.guestOs = guestOs; } public Hypervisor.HypervisorType getHypervisorType() { @@ -1111,6 +1144,10 @@ public class SystemVmTemplateRegistration { return arch; } + public String getGuestOs() { + return guestOs; + } + public String getDownloadedFilePath() { return downloadedFilePath; } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42000to42010.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42000to42010.java index d442dc89904..0c0a9f070ba 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42000to42010.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42000to42010.java @@ -100,8 +100,6 @@ public class Upgrade42000to42010 extends DbUpgradeAbstractImpl implements DbUpgr DbUpgradeUtils.addIndexIfNeeded(conn, "network_offering_details", "name"); - DbUpgradeUtils.addIndexIfNeeded(conn, "network_offering_details", "resource_id", "resource_type"); - DbUpgradeUtils.addIndexIfNeeded(conn, "service_offering", "cpu"); DbUpgradeUtils.addIndexIfNeeded(conn, "service_offering", "speed"); DbUpgradeUtils.addIndexIfNeeded(conn, "service_offering", "ram_size"); diff --git a/engine/schema/src/main/java/com/cloud/usage/dao/UsageJobDao.java b/engine/schema/src/main/java/com/cloud/usage/dao/UsageJobDao.java index f22a906054d..d4038d4ceeb 100644 --- a/engine/schema/src/main/java/com/cloud/usage/dao/UsageJobDao.java +++ b/engine/schema/src/main/java/com/cloud/usage/dao/UsageJobDao.java @@ -37,4 +37,6 @@ public interface UsageJobDao extends GenericDao { UsageJobVO isOwner(String hostname, int pid); void updateJobSuccess(Long jobId, long startMillis, long endMillis, long execTime, boolean success); + + void removeLastOpenJobsOwned(String hostname, int pid); } diff --git a/engine/schema/src/main/java/com/cloud/usage/dao/UsageJobDaoImpl.java b/engine/schema/src/main/java/com/cloud/usage/dao/UsageJobDaoImpl.java index 6d460aadd09..44a7d1a8b72 100644 --- a/engine/schema/src/main/java/com/cloud/usage/dao/UsageJobDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/usage/dao/UsageJobDaoImpl.java @@ -22,6 +22,7 @@ import java.util.Date; import java.util.List; +import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Component; import com.cloud.usage.UsageJobVO; @@ -114,7 +115,7 @@ public class UsageJobDaoImpl extends GenericDaoBase implements public UsageJobVO isOwner(String hostname, int pid) { TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.USAGE_DB); try { - if ((hostname == null) || (pid <= 0)) { + if (hostname == null || pid <= 0) { return null; } @@ -174,7 +175,7 @@ public class UsageJobDaoImpl extends GenericDaoBase implements SearchCriteria sc = createSearchCriteria(); sc.addAnd("endMillis", SearchCriteria.Op.EQ, Long.valueOf(0)); sc.addAnd("jobType", SearchCriteria.Op.EQ, Integer.valueOf(UsageJobVO.JOB_TYPE_SINGLE)); - sc.addAnd("scheduled", SearchCriteria.Op.EQ, Integer.valueOf(0)); + sc.addAnd("scheduled", SearchCriteria.Op.EQ, Integer.valueOf(UsageJobVO.JOB_NOT_SCHEDULED)); List jobs = search(sc, filter); if ((jobs == null) || jobs.isEmpty()) { @@ -194,4 +195,36 @@ public class UsageJobDaoImpl extends GenericDaoBase implements } return jobs.get(0).getHeartbeat(); } + + private List getLastOpenJobsOwned(String hostname, int pid) { + SearchCriteria sc = createSearchCriteria(); + sc.addAnd("endMillis", SearchCriteria.Op.EQ, Long.valueOf(0)); + sc.addAnd("host", SearchCriteria.Op.EQ, hostname); + if (pid > 0) { + sc.addAnd("pid", SearchCriteria.Op.EQ, Integer.valueOf(pid)); + } + return listBy(sc); + } + + @Override + public void removeLastOpenJobsOwned(String hostname, int pid) { + if (hostname == null) { + return; + } + + TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.USAGE_DB); + try { + List jobs = getLastOpenJobsOwned(hostname, pid); + if (CollectionUtils.isNotEmpty(jobs)) { + logger.info("Found {} opens job, to remove", jobs.size()); + for (UsageJobVO job : jobs) { + logger.debug("Removing job - id: {}, pid: {}, job type: {}, scheduled: {}, heartbeat: {}", + job.getId(), job.getPid(), job.getJobType(), job.getScheduled(), job.getHeartbeat()); + remove(job.getId()); + } + } + } finally { + txn.close(); + } + } } diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41910to42000.sql b/engine/schema/src/main/resources/META-INF/db/schema-41910to42000.sql index c36b71c2f25..eec4ac3f028 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41910to42000.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41910to42000.sql @@ -55,7 +55,7 @@ UPDATE `cloud`.`service_offering` SET ram_size = 512 WHERE unique_name IN ("Clou AND system_use = 1 AND ram_size < 512; -- NSX Plugin -- -CREATE TABLE `cloud`.`nsx_providers` ( +CREATE TABLE IF NOT EXISTS `cloud`.`nsx_providers` ( `id` bigint unsigned NOT NULL auto_increment COMMENT 'id', `uuid` varchar(40), `zone_id` bigint unsigned NOT NULL COMMENT 'Zone ID', diff --git a/engine/schema/src/test/java/com/cloud/storage/dao/VMTemplateDaoImplTest.java b/engine/schema/src/test/java/com/cloud/storage/dao/VMTemplateDaoImplTest.java index df0b36ebdbf..e97a887cc47 100644 --- a/engine/schema/src/test/java/com/cloud/storage/dao/VMTemplateDaoImplTest.java +++ b/engine/schema/src/test/java/com/cloud/storage/dao/VMTemplateDaoImplTest.java @@ -31,6 +31,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InjectMocks; @@ -186,4 +187,24 @@ public class VMTemplateDaoImplTest { VMTemplateVO result = templateDao.findLatestTemplateByTypeAndHypervisorAndArch(hypervisorType, arch, type); assertNull(result); } + + @Test + public void testFindSystemVMReadyTemplate() { + Long zoneId = 1L; + VMTemplateVO systemVmTemplate1 = mock(VMTemplateVO.class); + Mockito.when(systemVmTemplate1.getArch()).thenReturn(CPU.CPUArch.x86); + VMTemplateVO systemVmTemplate2 = mock(VMTemplateVO.class); + Mockito.when(systemVmTemplate2.getArch()).thenReturn(CPU.CPUArch.x86); + VMTemplateVO systemVmTemplate3 = mock(VMTemplateVO.class); + Mockito.when(systemVmTemplate3.getArch()).thenReturn(CPU.CPUArch.arm64); + Mockito.when(systemVmTemplate3.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM); + List templates = Arrays.asList(systemVmTemplate1, systemVmTemplate2, systemVmTemplate3); + Mockito.when(hostDao.listDistinctHypervisorTypes(zoneId)).thenReturn(Arrays.asList(Hypervisor.HypervisorType.KVM)); + SearchBuilder sb = mock(SearchBuilder.class); + templateDao.readySystemTemplateSearch = sb; + when(sb.create()).thenReturn(mock(SearchCriteria.class)); + doReturn(templates).when(templateDao).listBy(any(SearchCriteria.class), any(Filter.class)); + VMTemplateVO readyTemplate = templateDao.findSystemVMReadyTemplate(zoneId, Hypervisor.HypervisorType.KVM, CPU.CPUArch.arm64.getType()); + Assert.assertEquals(CPU.CPUArch.arm64, readyTemplate.getArch()); + } } diff --git a/engine/schema/src/test/java/com/cloud/upgrade/SystemVmTemplateRegistrationTest.java b/engine/schema/src/test/java/com/cloud/upgrade/SystemVmTemplateRegistrationTest.java index 27656fd33b0..6573a5565f3 100644 --- a/engine/schema/src/test/java/com/cloud/upgrade/SystemVmTemplateRegistrationTest.java +++ b/engine/schema/src/test/java/com/cloud/upgrade/SystemVmTemplateRegistrationTest.java @@ -192,7 +192,7 @@ public class SystemVmTemplateRegistrationTest { public void testValidateTemplateFile_fileNotFound() { SystemVmTemplateRegistration.MetadataTemplateDetails details = new SystemVmTemplateRegistration.MetadataTemplateDetails(Hypervisor.HypervisorType.KVM, - "name", "file", "url", "checksum", CPU.CPUArch.amd64); + "name", "file", "url", "checksum", CPU.CPUArch.amd64, "guestos"); SystemVmTemplateRegistration.NewTemplateMap.put(SystemVmTemplateRegistration.getHypervisorArchKey( details.getHypervisorType(), details.getArch()), details); doReturn(null).when(systemVmTemplateRegistration).getTemplateFile(details); @@ -209,7 +209,7 @@ public class SystemVmTemplateRegistrationTest { public void testValidateTemplateFile_checksumMismatch() { SystemVmTemplateRegistration.MetadataTemplateDetails details = new SystemVmTemplateRegistration.MetadataTemplateDetails(Hypervisor.HypervisorType.KVM, - "name", "file", "url", "checksum", CPU.CPUArch.amd64); + "name", "file", "url", "checksum", CPU.CPUArch.amd64, "guestos"); File dummyFile = new File("dummy.txt"); SystemVmTemplateRegistration.NewTemplateMap.put(SystemVmTemplateRegistration.getHypervisorArchKey( details.getHypervisorType(), details.getArch()), details); @@ -228,7 +228,7 @@ public class SystemVmTemplateRegistrationTest { public void testValidateTemplateFile_success() { SystemVmTemplateRegistration.MetadataTemplateDetails details = new SystemVmTemplateRegistration.MetadataTemplateDetails(Hypervisor.HypervisorType.KVM, - "name", "file", "url", "checksum", CPU.CPUArch.amd64); + "name", "file", "url", "checksum", CPU.CPUArch.amd64, "guestos"); File dummyFile = new File("dummy.txt"); SystemVmTemplateRegistration.NewTemplateMap.put(SystemVmTemplateRegistration.getHypervisorArchKey( details.getHypervisorType(), details.getArch()), details); @@ -285,7 +285,7 @@ public class SystemVmTemplateRegistrationTest { SystemVmTemplateRegistration.MetadataTemplateDetails details = new SystemVmTemplateRegistration.MetadataTemplateDetails(Hypervisor.HypervisorType.KVM, "name", "nonexistent.qcow2", "http://example.com/file.qcow2", - "", CPU.CPUArch.arm64); + "", CPU.CPUArch.arm64, "guestos"); try (MockedStatic filesMock = Mockito.mockStatic(Files.class); MockedStatic httpMock = Mockito.mockStatic(HttpUtils.class)) { filesMock.when(() -> Files.isWritable(any(Path.class))).thenReturn(true); @@ -301,7 +301,7 @@ public class SystemVmTemplateRegistrationTest { SystemVmTemplateRegistration.MetadataTemplateDetails details = new SystemVmTemplateRegistration.MetadataTemplateDetails(Hypervisor.HypervisorType.KVM, "name", "file.qcow2", "http://example.com/file.qcow2", - "", CPU.CPUArch.arm64); + "", CPU.CPUArch.arm64, "guestos"); try (MockedStatic filesMock = Mockito.mockStatic(Files.class); MockedStatic httpMock = Mockito.mockStatic(HttpUtils.class)) { filesMock.when(() -> Files.isWritable(any(Path.class))).thenReturn(false); diff --git a/engine/schema/templateConfig.sh b/engine/schema/templateConfig.sh index bed51a48a8f..d10b8668b12 100644 --- a/engine/schema/templateConfig.sh +++ b/engine/schema/templateConfig.sh @@ -42,6 +42,15 @@ function getGenericName() { fi } +function getGuestOS() { + hypervisor=$(echo "$1" | tr "[:upper:]" "[:lower:]") + if [[ "$hypervisor" == "vmware" || "$hypervisor" == "xenserver" ]]; then + echo "Other Linux (64-bit)" + else + echo "Debian GNU/Linux 12 (64-bit)" + fi +} + function getChecksum() { local fileData="$1" local hvName=$2 @@ -60,13 +69,14 @@ function createMetadataFile() { section="${template%%:*}" sectionHv="${section%%-*}" hvName=$(getGenericName $sectionHv) + guestos=$(getGuestOS $sectionHv) downloadurl="${template#*:}" arch=$(echo ${downloadurl#*"/systemvmtemplate-$VERSION-"} | cut -d'-' -f 1) templatename="systemvm-${sectionHv%.*}-${VERSION}-${arch}" checksum=$(getChecksum "$fileData" "$VERSION-${arch}-$hvName") filename=$(echo ${downloadurl##*'/'}) - echo -e "["$section"]\ntemplatename = $templatename\nchecksum = $checksum\ndownloadurl = $downloadurl\nfilename = $filename\narch = $arch\n" >> $METADATAFILE + echo -e "["$section"]\ntemplatename = $templatename\nchecksum = $checksum\ndownloadurl = $downloadurl\nfilename = $filename\narch = $arch\nguestos = $guestos\n" >> $METADATAFILE done } diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotObject.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotObject.java index a3964bd461e..7602a142f88 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotObject.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotObject.java @@ -172,10 +172,15 @@ public class SnapshotObject implements SnapshotInfo { @Override public long getPhysicalSize() { long physicalSize = 0; - SnapshotDataStoreVO snapshotStore = snapshotStoreDao.findByStoreSnapshot(DataStoreRole.Image, store.getId(), snapshot.getId()); - if (snapshotStore != null) { - physicalSize = snapshotStore.getPhysicalSize(); + for (DataStoreRole role : List.of(DataStoreRole.Image, DataStoreRole.Primary)) { + logger.trace("Retrieving snapshot [{}] size from {} storage.", snapshot.getUuid(), role); + SnapshotDataStoreVO snapshotStore = snapshotStoreDao.findByStoreSnapshot(role, store.getId(), snapshot.getId()); + if (snapshotStore != null) { + return snapshotStore.getPhysicalSize(); + } + logger.trace("Snapshot [{}] size not found on {} storage.", snapshot.getUuid(), role); } + logger.warn("Snapshot [{}] reference not found in any storage. There may be an inconsistency on the database.", snapshot.getUuid()); return physicalSize; } diff --git a/packaging/el8/cloud.spec b/packaging/el8/cloud.spec index 22fede6fb85..1e862ba0dc0 100644 --- a/packaging/el8/cloud.spec +++ b/packaging/el8/cloud.spec @@ -102,7 +102,7 @@ The Apache CloudStack files shared between agent and management server Summary: CloudStack Agent for KVM hypervisors Requires: (openssh-clients or openssh) Requires: java-17-openjdk -Requires: tzdata-java +Requires: (tzdata-java or timezone-java) Requires: %{name}-common = %{_ver} Requires: libvirt Requires: libvirt-daemon-driver-storage-rbd @@ -143,7 +143,7 @@ The CloudStack baremetal agent %package usage Summary: CloudStack Usage calculation server Requires: java-17-openjdk -Requires: tzdata-java +Requires: (tzdata-java or timezone-java) Group: System Environment/Libraries %description usage The CloudStack usage calculation service diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java index 141a0073498..9b5672e228f 100644 --- a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java @@ -51,6 +51,7 @@ import javax.inject.Inject; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.Date; import java.util.List; import java.util.Locale; @@ -162,6 +163,7 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co if (VirtualMachine.State.Stopped.equals(vm.getState())) { List vmVolumes = volumeDao.findByInstance(vm.getId()); + vmVolumes.sort(Comparator.comparing(Volume::getDeviceId)); List volumePaths = getVolumePaths(vmVolumes); command.setVolumePaths(volumePaths); } @@ -212,7 +214,10 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co @Override public boolean restoreVMFromBackup(VirtualMachine vm, Backup backup) { List backedVolumes = backup.getBackedUpVolumes(); - List volumes = backedVolumes.stream().map(volume -> volumeDao.findByUuid(volume.getUuid())).collect(Collectors.toList()); + List volumes = backedVolumes.stream() + .map(volume -> volumeDao.findByUuid(volume.getUuid())) + .sorted((v1, v2) -> Long.compare(v1.getDeviceId(), v2.getDeviceId())) + .collect(Collectors.toList()); LOG.debug("Restoring vm {} from backup {} on the NAS Backup Provider", vm, backup); BackupRepository backupRepository = getBackupRepository(vm, backup); @@ -246,9 +251,13 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co if (Objects.isNull(storagePool)) { throw new CloudRuntimeException("Unable to find storage pool associated to the volume"); } - String volumePathPrefix = String.format("/mnt/%s", storagePool.getUuid()); + String volumePathPrefix; if (ScopeType.HOST.equals(storagePool.getScope())) { volumePathPrefix = storagePool.getPath(); + } else if (Storage.StoragePoolType.SharedMountPoint.equals(storagePool.getPoolType())) { + volumePathPrefix = storagePool.getPath(); + } else { + volumePathPrefix = String.format("/mnt/%s", storagePool.getUuid()); } volumePaths.add(String.format("%s/%s", volumePathPrefix, volume.getPath())); } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java index 26b8de53083..0602fd6322f 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java @@ -250,6 +250,15 @@ public class BridgeVifDriver extends VifDriverBase { intf.defBridgeNet(_bridges.get("private"), null, nic.getMac(), getGuestNicModel(guestOsType, nicAdapter)); } else if (nic.getType() == Networks.TrafficType.Storage) { String storageBrName = nic.getName() == null ? _bridges.get("private") : nic.getName(); + if (nic.getBroadcastType() == Networks.BroadcastDomainType.Storage) { + vNetId = Networks.BroadcastDomainType.getValue(nic.getBroadcastUri()); + protocol = Networks.BroadcastDomainType.Vlan.scheme(); + } + if (isValidProtocolAndVnetId(vNetId, protocol)) { + logger.debug(String.format("creating a vNet dev and bridge for %s traffic per traffic label %s", + Networks.TrafficType.Storage.name(), trafficLabel)); + storageBrName = createVnetBr(vNetId, storageBrName, protocol); + } intf.defBridgeNet(storageBrName, null, nic.getMac(), getGuestNicModel(guestOsType, nicAdapter)); } if (nic.getPxeDisable()) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 13518de5cb3..19c6e7145a6 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -17,6 +17,8 @@ package com.cloud.hypervisor.kvm.resource; import static com.cloud.host.Host.HOST_INSTANCE_CONVERSION; +import static com.cloud.host.Host.HOST_OVFTOOL_VERSION; +import static com.cloud.host.Host.HOST_VIRTV2V_VERSION; import static com.cloud.host.Host.HOST_VOLUME_ENCRYPTION; import java.io.BufferedReader; @@ -3366,7 +3368,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv if (!meetRequirements) { return false; } - return isUbuntuHost() || isIoUringSupportedByQemu(); + return isUbuntuOrDebianHost() || isIoUringSupportedByQemu(); } /** @@ -3379,13 +3381,14 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv return diskBus != DiskDef.DiskBus.IDE || getHypervisorQemuVersion() >= HYPERVISOR_QEMU_VERSION_IDE_DISCARD_FIXED; } - public boolean isUbuntuHost() { + public boolean isUbuntuOrDebianHost() { Map versionString = getVersionStrings(); String hostKey = "Host.OS"; if (MapUtils.isEmpty(versionString) || !versionString.containsKey(hostKey) || versionString.get(hostKey) == null) { return false; } - return versionString.get(hostKey).equalsIgnoreCase("ubuntu"); + return versionString.get(hostKey).equalsIgnoreCase("ubuntu") + || versionString.get(hostKey).toLowerCase().startsWith("debian"); } private KVMPhysicalDisk getPhysicalDiskFromNfsStore(String dataStoreUrl, DataTO data) { @@ -3503,10 +3506,10 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv public synchronized String attachOrDetachISO(final Connect conn, final String vmName, String isoPath, final boolean isAttach, final Integer diskSeq) throws LibvirtException, URISyntaxException, InternalErrorException { final DiskDef iso = new DiskDef(); - if (isAttach && StringUtils.isNotBlank(isoPath) && isoPath.lastIndexOf("/") > 0) { - if (isoPath.startsWith(getConfigPath() + "/" + ConfigDrive.CONFIGDRIVEDIR) && isoPath.contains(vmName)) { + if (isAttach && StringUtils.isNotBlank(isoPath)) { + if (isoPath.startsWith(getConfigPath() + "/" + ConfigDrive.CONFIGDRIVEDIR) || isoPath.contains(vmName)) { iso.defISODisk(isoPath, diskSeq, DiskDef.DiskType.FILE); - } else { + } else if (isoPath.lastIndexOf("/") > 0) { final int index = isoPath.lastIndexOf("/"); final String path = isoPath.substring(0, index); final String name = isoPath.substring(index + 1); @@ -3530,7 +3533,6 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv cleanupDisk(disk); } } - } return result; } @@ -3766,7 +3768,14 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv cmd.setIqn(getIqn()); cmd.getHostDetails().put(HOST_VOLUME_ENCRYPTION, String.valueOf(hostSupportsVolumeEncryption())); cmd.setHostTags(getHostTags()); - cmd.getHostDetails().put(HOST_INSTANCE_CONVERSION, String.valueOf(hostSupportsInstanceConversion())); + boolean instanceConversionSupported = hostSupportsInstanceConversion(); + cmd.getHostDetails().put(HOST_INSTANCE_CONVERSION, String.valueOf(instanceConversionSupported)); + if (instanceConversionSupported) { + cmd.getHostDetails().put(HOST_VIRTV2V_VERSION, getHostVirtV2vVersion()); + } + if (hostSupportsOvfExport()) { + cmd.getHostDetails().put(HOST_OVFTOOL_VERSION, getHostOvfToolVersion()); + } HealthCheckResult healthCheckResult = getHostHealthCheckResult(); if (healthCheckResult != HealthCheckResult.IGNORE) { cmd.setHostHealthCheckResult(healthCheckResult == HealthCheckResult.SUCCESS); @@ -5348,14 +5357,14 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv public boolean hostSupportsInstanceConversion() { int exitValue = Script.runSimpleBashScriptForExitValue(INSTANCE_CONVERSION_SUPPORTED_CHECK_CMD); - if (isUbuntuHost() && exitValue == 0) { + if (isUbuntuOrDebianHost() && exitValue == 0) { exitValue = Script.runSimpleBashScriptForExitValue(UBUNTU_NBDKIT_PKG_CHECK_CMD); } return exitValue == 0; } public boolean hostSupportsWindowsGuestConversion() { - if (isUbuntuHost()) { + if (isUbuntuOrDebianHost()) { int exitValue = Script.runSimpleBashScriptForExitValue(UBUNTU_WINDOWS_GUEST_CONVERSION_SUPPORTED_CHECK_CMD); return exitValue == 0; } @@ -5368,8 +5377,24 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv return exitValue == 0; } + public String getHostVirtV2vVersion() { + if (!hostSupportsInstanceConversion()) { + return ""; + } + String cmd = String.format("%s | awk '{print $2}'", INSTANCE_CONVERSION_SUPPORTED_CHECK_CMD); + String version = Script.runSimpleBashScript(cmd); + return StringUtils.isNotBlank(version) ? version.split(",")[0] : ""; + } + + public String getHostOvfToolVersion() { + if (!hostSupportsOvfExport()) { + return ""; + } + return Script.runSimpleBashScript(OVF_EXPORT_TOOl_GET_VERSION_CMD); + } + public boolean ovfExportToolSupportsParallelThreads() { - String ovfExportToolVersion = Script.runSimpleBashScript(OVF_EXPORT_TOOl_GET_VERSION_CMD); + String ovfExportToolVersion = getHostOvfToolVersion(); if (StringUtils.isBlank(ovfExportToolVersion)) { return false; } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java index 93ad084b437..1383190933d 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java @@ -249,9 +249,7 @@ public class LibvirtVMDef { guestDef.append("\n"); } } - if (_arch == null || !_arch.equals("aarch64")) { - guestDef.append("\n"); - } + guestDef.append("\n"); guestDef.append("\n"); if (iothreads) { guestDef.append(String.format("%s", NUMBER_OF_IOTHREADS)); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckConvertInstanceCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckConvertInstanceCommandWrapper.java index d3ebb28b106..b94b4830003 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckConvertInstanceCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckConvertInstanceCommandWrapper.java @@ -32,7 +32,7 @@ public class LibvirtCheckConvertInstanceCommandWrapper extends CommandWrapper { + private static final List STORAGE_POOL_TYPES_SUPPORTED = Arrays.asList(Storage.StoragePoolType.Filesystem, Storage.StoragePoolType.NetworkFilesystem); + @Override public Answer execute(final CheckVolumeCommand command, final LibvirtComputingResource libvirtComputingResource) { String result = null; @@ -50,34 +58,76 @@ public final class LibvirtCheckVolumeCommandWrapper extends CommandWrapper getVolumeDetails(KVMStoragePool pool, KVMPhysicalDisk disk) { + Map info = getDiskFileInfo(pool, disk, true); + if (MapUtils.isEmpty(info)) { + return null; + } + + Map volumeDetails = new HashMap<>(); + + String backingFilePath = info.get(QemuImg.BACKING_FILE); + if (StringUtils.isNotBlank(backingFilePath)) { + volumeDetails.put(VolumeOnStorageTO.Detail.BACKING_FILE, backingFilePath); + } + String backingFileFormat = info.get(QemuImg.BACKING_FILE_FORMAT); + if (StringUtils.isNotBlank(backingFileFormat)) { + volumeDetails.put(VolumeOnStorageTO.Detail.BACKING_FILE_FORMAT, backingFileFormat); + } + String clusterSize = info.get(QemuImg.CLUSTER_SIZE); + if (StringUtils.isNotBlank(clusterSize)) { + volumeDetails.put(VolumeOnStorageTO.Detail.CLUSTER_SIZE, clusterSize); + } + String fileFormat = info.get(QemuImg.FILE_FORMAT); + if (StringUtils.isNotBlank(fileFormat)) { + volumeDetails.put(VolumeOnStorageTO.Detail.FILE_FORMAT, fileFormat); + } + String encrypted = info.get(QemuImg.ENCRYPTED); + if (StringUtils.isNotBlank(encrypted) && encrypted.equalsIgnoreCase("yes")) { + volumeDetails.put(VolumeOnStorageTO.Detail.IS_ENCRYPTED, String.valueOf(Boolean.TRUE)); + } + Boolean isLocked = isDiskFileLocked(pool, disk); + volumeDetails.put(VolumeOnStorageTO.Detail.IS_LOCKED, String.valueOf(isLocked)); + + return volumeDetails; + } + + private Map getDiskFileInfo(KVMStoragePool pool, KVMPhysicalDisk disk, boolean secure) { + if (!STORAGE_POOL_TYPES_SUPPORTED.contains(pool.getType())) { + return new HashMap<>(); // unknown + } try { QemuImg qemu = new QemuImg(0); - QemuImgFile qemuFile = new QemuImgFile(path); - Map info = qemu.info(qemuFile); - if (info.containsKey(QemuImg.VIRTUAL_SIZE)) { - return Long.parseLong(info.get(QemuImg.VIRTUAL_SIZE)); - } else { - throw new CloudRuntimeException("Unable to determine virtual size of volume at path " + path); - } + QemuImgFile qemuFile = new QemuImgFile(disk.getPath(), disk.getFormat()); + return qemu.info(qemuFile, secure); } catch (QemuImgException | LibvirtException ex) { - throw new CloudRuntimeException("Error when inspecting volume at path " + path, ex); + logger.error("Failed to get info of disk file: " + ex.getMessage()); + return null; } } + + private boolean isDiskFileLocked(KVMStoragePool pool, KVMPhysicalDisk disk) { + Map info = getDiskFileInfo(pool, disk, false); + return info == null; + } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapper.java index a11730a1240..abc408f20f6 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapper.java @@ -18,22 +18,12 @@ // package com.cloud.hypervisor.kvm.resource.wrapper; -import java.io.BufferedInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; import java.net.URLEncoder; import java.nio.charset.Charset; -import java.util.ArrayList; import java.util.List; import java.util.UUID; -import java.util.stream.Collectors; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; -import org.apache.cloudstack.vm.UnmanagedInstanceTO; -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import com.cloud.agent.api.Answer; @@ -44,17 +34,12 @@ import com.cloud.agent.api.to.NfsTO; import com.cloud.agent.api.to.RemoteInstanceTO; import com.cloud.hypervisor.Hypervisor; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; -import com.cloud.hypervisor.kvm.resource.LibvirtDomainXMLParser; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef; -import com.cloud.hypervisor.kvm.storage.KVMPhysicalDisk; import com.cloud.hypervisor.kvm.storage.KVMStoragePool; import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; -import com.cloud.storage.Storage; import com.cloud.utils.FileUtil; -import com.cloud.utils.Pair; -import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.OutputInterpreter; import com.cloud.utils.script.Script; @@ -75,9 +60,9 @@ public class LibvirtConvertInstanceCommandWrapper extends CommandWrapper getTemporaryDisksFromParsedXml(KVMStoragePool pool, LibvirtDomainXMLParser xmlParser, String convertedBasePath) { - List disksDefs = xmlParser.getDisks(); - disksDefs = disksDefs.stream().filter(x -> x.getDiskType() == LibvirtVMDef.DiskDef.DiskType.FILE && - x.getDeviceType() == LibvirtVMDef.DiskDef.DeviceType.DISK).collect(Collectors.toList()); - if (CollectionUtils.isEmpty(disksDefs)) { - String err = String.format("Cannot find any disk defined on the converted XML domain %s.xml", convertedBasePath); - logger.error(err); - throw new CloudRuntimeException(err); - } - sanitizeDisksPath(disksDefs); - return getPhysicalDisksFromDefPaths(disksDefs, pool); - } - - private List getPhysicalDisksFromDefPaths(List disksDefs, KVMStoragePool pool) { - List disks = new ArrayList<>(); - for (LibvirtVMDef.DiskDef diskDef : disksDefs) { - KVMPhysicalDisk physicalDisk = pool.getPhysicalDisk(diskDef.getDiskPath()); - disks.add(physicalDisk); - } - return disks; - } - - protected List getTemporaryDisksWithPrefixFromTemporaryPool(KVMStoragePool pool, String path, String prefix) { - String msg = String.format("Could not parse correctly the converted XML domain, checking for disks on %s with prefix %s", path, prefix); - logger.info(msg); - pool.refresh(); - List disksWithPrefix = pool.listPhysicalDisks() - .stream() - .filter(x -> x.getName().startsWith(prefix) && !x.getName().endsWith(".xml")) - .collect(Collectors.toList()); - if (CollectionUtils.isEmpty(disksWithPrefix)) { - msg = String.format("Could not find any converted disk with prefix %s on temporary location %s", prefix, path); - logger.error(msg); - throw new CloudRuntimeException(msg); - } - return disksWithPrefix; - } - - private void cleanupDisksAndDomainFromTemporaryLocation(List disks, - KVMStoragePool temporaryStoragePool, - String temporaryConvertUuid) { - for (KVMPhysicalDisk disk : disks) { - logger.info(String.format("Cleaning up temporary disk %s after conversion from temporary location", disk.getName())); - temporaryStoragePool.deletePhysicalDisk(disk.getName(), Storage.ImageFormat.QCOW2); - } - logger.info(String.format("Cleaning up temporary domain %s after conversion from temporary location", temporaryConvertUuid)); - FileUtil.deleteFiles(temporaryStoragePool.getLocalPath(), temporaryConvertUuid, ".xml"); - } - protected void sanitizeDisksPath(List disks) { for (LibvirtVMDef.DiskDef disk : disks) { String[] diskPathParts = disk.getDiskPath().split("/"); @@ -262,114 +198,6 @@ public class LibvirtConvertInstanceCommandWrapper extends CommandWrapper moveTemporaryDisksToDestination(List temporaryDisks, - List destinationStoragePools, - KVMStoragePoolManager storagePoolMgr) { - List targetDisks = new ArrayList<>(); - if (temporaryDisks.size() != destinationStoragePools.size()) { - String warn = String.format("Discrepancy between the converted instance disks (%s) " + - "and the expected number of disks (%s)", temporaryDisks.size(), destinationStoragePools.size()); - logger.warn(warn); - } - for (int i = 0; i < temporaryDisks.size(); i++) { - String poolPath = destinationStoragePools.get(i); - KVMStoragePool destinationPool = storagePoolMgr.getStoragePool(Storage.StoragePoolType.NetworkFilesystem, poolPath); - if (destinationPool == null) { - String err = String.format("Could not find a storage pool by URI: %s", poolPath); - logger.error(err); - continue; - } - if (destinationPool.getType() != Storage.StoragePoolType.NetworkFilesystem) { - String err = String.format("Storage pool by URI: %s is not an NFS storage", poolPath); - logger.error(err); - continue; - } - KVMPhysicalDisk sourceDisk = temporaryDisks.get(i); - if (logger.isDebugEnabled()) { - String msg = String.format("Trying to copy converted instance disk number %s from the temporary location %s" + - " to destination storage pool %s", i, sourceDisk.getPool().getLocalPath(), destinationPool.getUuid()); - logger.debug(msg); - } - - String destinationName = UUID.randomUUID().toString(); - - KVMPhysicalDisk destinationDisk = storagePoolMgr.copyPhysicalDisk(sourceDisk, destinationName, destinationPool, 7200 * 1000); - targetDisks.add(destinationDisk); - } - return targetDisks; - } - - private UnmanagedInstanceTO getConvertedUnmanagedInstance(String baseName, - List vmDisks, - LibvirtDomainXMLParser xmlParser) { - UnmanagedInstanceTO instanceTO = new UnmanagedInstanceTO(); - instanceTO.setName(baseName); - instanceTO.setDisks(getUnmanagedInstanceDisks(vmDisks, xmlParser)); - instanceTO.setNics(getUnmanagedInstanceNics(xmlParser)); - return instanceTO; - } - - private List getUnmanagedInstanceNics(LibvirtDomainXMLParser xmlParser) { - List nics = new ArrayList<>(); - if (xmlParser != null) { - List interfaces = xmlParser.getInterfaces(); - for (LibvirtVMDef.InterfaceDef interfaceDef : interfaces) { - UnmanagedInstanceTO.Nic nic = new UnmanagedInstanceTO.Nic(); - nic.setMacAddress(interfaceDef.getMacAddress()); - nic.setNicId(interfaceDef.getBrName()); - nic.setAdapterType(interfaceDef.getModel().toString()); - nics.add(nic); - } - } - return nics; - } - - protected List getUnmanagedInstanceDisks(List vmDisks, LibvirtDomainXMLParser xmlParser) { - List instanceDisks = new ArrayList<>(); - List diskDefs = xmlParser != null ? xmlParser.getDisks() : null; - for (int i = 0; i< vmDisks.size(); i++) { - KVMPhysicalDisk physicalDisk = vmDisks.get(i); - KVMStoragePool storagePool = physicalDisk.getPool(); - UnmanagedInstanceTO.Disk disk = new UnmanagedInstanceTO.Disk(); - disk.setPosition(i); - Pair storagePoolHostAndPath = getNfsStoragePoolHostAndPath(storagePool); - disk.setDatastoreHost(storagePoolHostAndPath.first()); - disk.setDatastorePath(storagePoolHostAndPath.second()); - disk.setDatastoreName(storagePool.getUuid()); - disk.setDatastoreType(storagePool.getType().name()); - disk.setCapacity(physicalDisk.getVirtualSize()); - disk.setFileBaseName(physicalDisk.getName()); - if (CollectionUtils.isNotEmpty(diskDefs)) { - LibvirtVMDef.DiskDef diskDef = diskDefs.get(i); - disk.setController(diskDef.getBusType() != null ? diskDef.getBusType().toString() : LibvirtVMDef.DiskDef.DiskBus.VIRTIO.toString()); - } else { - // If the job is finished but we cannot parse the XML, the guest VM can use the virtio driver - disk.setController(LibvirtVMDef.DiskDef.DiskBus.VIRTIO.toString()); - } - instanceDisks.add(disk); - } - return instanceDisks; - } - - protected Pair getNfsStoragePoolHostAndPath(KVMStoragePool storagePool) { - String sourceHostIp = null; - String sourcePath = null; - List commands = new ArrayList<>(); - commands.add(new String[]{Script.getExecutableAbsolutePath("mount")}); - commands.add(new String[]{Script.getExecutableAbsolutePath("grep"), storagePool.getLocalPath()}); - String storagePoolMountPoint = Script.executePipedCommands(commands, 0).second(); - logger.debug(String.format("NFS Storage pool: %s - local path: %s, mount point: %s", storagePool.getUuid(), storagePool.getLocalPath(), storagePoolMountPoint)); - if (StringUtils.isNotEmpty(storagePoolMountPoint)) { - String[] res = storagePoolMountPoint.strip().split(" "); - res = res[0].split(":"); - if (res.length > 1) { - sourceHostIp = res[0].strip(); - sourcePath = res[1].strip(); - } - } - return new Pair<>(sourceHostIp, sourcePath); - } - private boolean exportOVAFromVMOnVcenter(String vmExportUrl, String targetOvfDir, int noOfThreads, @@ -412,27 +240,6 @@ public class LibvirtConvertInstanceCommandWrapper extends CommandWrapper { + private static final List STORAGE_POOL_TYPES_SUPPORTED = Arrays.asList(Storage.StoragePoolType.Filesystem, Storage.StoragePoolType.NetworkFilesystem); @Override public Answer execute(final CopyRemoteVolumeCommand command, final LibvirtComputingResource libvirtComputingResource) { @@ -55,14 +62,19 @@ public final class LibvirtCopyRemoteVolumeCommandWrapper extends CommandWrapper< int timeoutInSecs = command.getWait(); try { - if (storageFilerTO.getType() == Storage.StoragePoolType.Filesystem || - storageFilerTO.getType() == Storage.StoragePoolType.NetworkFilesystem) { + if (STORAGE_POOL_TYPES_SUPPORTED.contains(storageFilerTO.getType())) { String filename = libvirtComputingResource.copyVolume(srcIp, username, password, dstPath, srcFile, tmpPath, timeoutInSecs); logger.debug("Volume " + srcFile + " copy successful, copied to file: " + filename); final KVMPhysicalDisk vol = pool.getPhysicalDisk(filename); final String path = vol.getPath(); - long size = getVirtualSizeFromFile(path); - return new CopyRemoteVolumeAnswer(command, "", filename, size); + try { + KVMPhysicalDisk.checkQcow2File(path); + } catch (final CloudRuntimeException e) { + return new CopyRemoteVolumeAnswer(command, false, "", filename, 0, getVolumeDetails(pool, vol)); + } + + long size = KVMPhysicalDisk.getVirtualSizeFromFile(path); + return new CopyRemoteVolumeAnswer(command, true, "", filename, size, getVolumeDetails(pool, vol)); } else { String msg = "Unsupported storage pool type: " + storageFilerTO.getType().toString() + ", only local and NFS pools are supported"; return new Answer(command, false, msg); @@ -74,18 +86,56 @@ public final class LibvirtCopyRemoteVolumeCommandWrapper extends CommandWrapper< } } - private long getVirtualSizeFromFile(String path) { + private Map getVolumeDetails(KVMStoragePool pool, KVMPhysicalDisk disk) { + Map info = getDiskFileInfo(pool, disk, true); + if (MapUtils.isEmpty(info)) { + return null; + } + + Map volumeDetails = new HashMap<>(); + + String backingFilePath = info.get(QemuImg.BACKING_FILE); + if (StringUtils.isNotBlank(backingFilePath)) { + volumeDetails.put(VolumeOnStorageTO.Detail.BACKING_FILE, backingFilePath); + } + String backingFileFormat = info.get(QemuImg.BACKING_FILE_FORMAT); + if (StringUtils.isNotBlank(backingFileFormat)) { + volumeDetails.put(VolumeOnStorageTO.Detail.BACKING_FILE_FORMAT, backingFileFormat); + } + String clusterSize = info.get(QemuImg.CLUSTER_SIZE); + if (StringUtils.isNotBlank(clusterSize)) { + volumeDetails.put(VolumeOnStorageTO.Detail.CLUSTER_SIZE, clusterSize); + } + String fileFormat = info.get(QemuImg.FILE_FORMAT); + if (StringUtils.isNotBlank(fileFormat)) { + volumeDetails.put(VolumeOnStorageTO.Detail.FILE_FORMAT, fileFormat); + } + String encrypted = info.get(QemuImg.ENCRYPTED); + if (StringUtils.isNotBlank(encrypted) && encrypted.equalsIgnoreCase("yes")) { + volumeDetails.put(VolumeOnStorageTO.Detail.IS_ENCRYPTED, String.valueOf(Boolean.TRUE)); + } + Boolean isLocked = isDiskFileLocked(pool, disk); + volumeDetails.put(VolumeOnStorageTO.Detail.IS_LOCKED, String.valueOf(isLocked)); + + return volumeDetails; + } + + private Map getDiskFileInfo(KVMStoragePool pool, KVMPhysicalDisk disk, boolean secure) { + if (!STORAGE_POOL_TYPES_SUPPORTED.contains(pool.getType())) { + return new HashMap<>(); // unknown + } try { QemuImg qemu = new QemuImg(0); - QemuImgFile qemuFile = new QemuImgFile(path); - Map info = qemu.info(qemuFile); - if (info.containsKey(QemuImg.VIRTUAL_SIZE)) { - return Long.parseLong(info.get(QemuImg.VIRTUAL_SIZE)); - } else { - throw new CloudRuntimeException("Unable to determine virtual size of volume at path " + path); - } + QemuImgFile qemuFile = new QemuImgFile(disk.getPath(), disk.getFormat()); + return qemu.info(qemuFile, secure); } catch (QemuImgException | LibvirtException ex) { - throw new CloudRuntimeException("Error when inspecting volume at path " + path, ex); + logger.error("Failed to get info of disk file: " + ex.getMessage()); + return null; } } + + private boolean isDiskFileLocked(KVMStoragePool pool, KVMPhysicalDisk disk) { + Map info = getDiskFileInfo(pool, disk, false); + return info == null; + } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java index 821a80f5cca..6facf169602 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java @@ -36,6 +36,7 @@ import org.apache.cloudstack.utils.qemu.QemuImg; import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; import org.apache.cloudstack.utils.qemu.QemuImgException; import org.apache.cloudstack.utils.qemu.QemuImgFile; +import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; import org.libvirt.LibvirtException; @@ -91,37 +92,46 @@ public final class LibvirtGetVolumesOnStorageCommandWrapper extends CommandWrapp if (disk.getQemuEncryptFormat() != null) { volumeOnStorageTO.setQemuEncryptFormat(disk.getQemuEncryptFormat().toString()); } - String backingFilePath = info.get(QemuImg.BACKING_FILE); - if (StringUtils.isNotBlank(backingFilePath)) { - volumeOnStorageTO.addDetail(VolumeOnStorageTO.Detail.BACKING_FILE, backingFilePath); - } - String backingFileFormat = info.get(QemuImg.BACKING_FILE_FORMAT); - if (StringUtils.isNotBlank(backingFileFormat)) { - volumeOnStorageTO.addDetail(VolumeOnStorageTO.Detail.BACKING_FILE_FORMAT, backingFileFormat); - } - String clusterSize = info.get(QemuImg.CLUSTER_SIZE); - if (StringUtils.isNotBlank(clusterSize)) { - volumeOnStorageTO.addDetail(VolumeOnStorageTO.Detail.CLUSTER_SIZE, clusterSize); - } String fileFormat = info.get(QemuImg.FILE_FORMAT); - if (StringUtils.isNotBlank(fileFormat)) { - if (!fileFormat.equalsIgnoreCase(disk.getFormat().toString())) { - return new GetVolumesOnStorageAnswer(command, false, String.format("The file format is %s, but expected to be %s", fileFormat, disk.getFormat())); - } - volumeOnStorageTO.addDetail(VolumeOnStorageTO.Detail.FILE_FORMAT, fileFormat); + if (StringUtils.isNotBlank(fileFormat) && !fileFormat.equalsIgnoreCase(disk.getFormat().toString())) { + return new GetVolumesOnStorageAnswer(command, false, String.format("The file format is %s, but expected to be %s", fileFormat, disk.getFormat())); } - String encrypted = info.get(QemuImg.ENCRYPTED); - if (StringUtils.isNotBlank(encrypted) && encrypted.equalsIgnoreCase("yes")) { - volumeOnStorageTO.addDetail(VolumeOnStorageTO.Detail.IS_ENCRYPTED, String.valueOf(Boolean.TRUE)); - } - Boolean isLocked = isDiskFileLocked(storagePool, disk); - volumeOnStorageTO.addDetail(VolumeOnStorageTO.Detail.IS_LOCKED, String.valueOf(isLocked)); + addDetailsToVolumeOnStorageTO(volumeOnStorageTO, info, storagePool, disk); volumes.add(volumeOnStorageTO); } return new GetVolumesOnStorageAnswer(command, volumes); } + private void addDetailsToVolumeOnStorageTO(VolumeOnStorageTO volumeOnStorageTO, final Map info, final KVMStoragePool storagePool, final KVMPhysicalDisk disk) { + if (MapUtils.isEmpty(info)) { + return; + } + + String backingFilePath = info.get(QemuImg.BACKING_FILE); + if (StringUtils.isNotBlank(backingFilePath)) { + volumeOnStorageTO.addDetail(VolumeOnStorageTO.Detail.BACKING_FILE, backingFilePath); + } + String backingFileFormat = info.get(QemuImg.BACKING_FILE_FORMAT); + if (StringUtils.isNotBlank(backingFileFormat)) { + volumeOnStorageTO.addDetail(VolumeOnStorageTO.Detail.BACKING_FILE_FORMAT, backingFileFormat); + } + String clusterSize = info.get(QemuImg.CLUSTER_SIZE); + if (StringUtils.isNotBlank(clusterSize)) { + volumeOnStorageTO.addDetail(VolumeOnStorageTO.Detail.CLUSTER_SIZE, clusterSize); + } + String fileFormat = info.get(QemuImg.FILE_FORMAT); + if (StringUtils.isNotBlank(fileFormat)) { + volumeOnStorageTO.addDetail(VolumeOnStorageTO.Detail.FILE_FORMAT, fileFormat); + } + String encrypted = info.get(QemuImg.ENCRYPTED); + if (StringUtils.isNotBlank(encrypted) && encrypted.equalsIgnoreCase("yes")) { + volumeOnStorageTO.addDetail(VolumeOnStorageTO.Detail.IS_ENCRYPTED, String.valueOf(Boolean.TRUE)); + } + Boolean isLocked = isDiskFileLocked(storagePool, disk); + volumeOnStorageTO.addDetail(VolumeOnStorageTO.Detail.IS_LOCKED, String.valueOf(isLocked)); + } + private GetVolumesOnStorageAnswer addAllVolumes(final GetVolumesOnStorageCommand command, final KVMStoragePool storagePool, String keyword) { List volumes = new ArrayList<>(); @@ -134,11 +144,21 @@ public final class LibvirtGetVolumesOnStorageCommandWrapper extends CommandWrapp if (!isDiskFormatSupported(disk)) { continue; } + Map info = getDiskFileInfo(storagePool, disk, true); + if (info == null) { + continue; + } VolumeOnStorageTO volumeOnStorageTO = new VolumeOnStorageTO(Hypervisor.HypervisorType.KVM, disk.getName(), disk.getName(), disk.getPath(), disk.getFormat().toString(), disk.getSize(), disk.getVirtualSize()); if (disk.getQemuEncryptFormat() != null) { volumeOnStorageTO.setQemuEncryptFormat(disk.getQemuEncryptFormat().toString()); } + String fileFormat = info.get(QemuImg.FILE_FORMAT); + if (StringUtils.isNotBlank(fileFormat) && !fileFormat.equalsIgnoreCase(disk.getFormat().toString())) { + continue; + } + addDetailsToVolumeOnStorageTO(volumeOnStorageTO, info, storagePool, disk); + volumes.add(volumeOnStorageTO); } return new GetVolumesOnStorageAnswer(command, volumes); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java index e15a3287692..2d3b58a809e 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java @@ -899,7 +899,7 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper hostDetails = new HashMap(); - if (hostSupportsUefi(libvirtComputingResource.isUbuntuHost()) && libvirtComputingResource.isUefiPropertiesFileLoaded()) { + if (hostSupportsUefi(libvirtComputingResource.isUbuntuOrDebianHost()) && libvirtComputingResource.isUefiPropertiesFileLoaded()) { hostDetails.put(Host.HOST_UEFI_ENABLE, Boolean.TRUE.toString()); } + if (libvirtComputingResource.hostSupportsInstanceConversion()) { + hostDetails.put(Host.HOST_VIRTV2V_VERSION, libvirtComputingResource.getHostVirtV2vVersion()); + } + + if (libvirtComputingResource.hostSupportsOvfExport()) { + hostDetails.put(Host.HOST_OVFTOOL_VERSION, libvirtComputingResource.getHostOvfToolVersion()); + } + return new ReadyAnswer(command, hostDetails); } - private boolean hostSupportsUefi(boolean isUbuntuHost) { + private boolean hostSupportsUefi(boolean isUbuntuOrDebianHost) { int timeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.AGENT_SCRIPT_TIMEOUT) * 1000; // Get property value & convert to milliseconds int result; - if (isUbuntuHost) { + if (isUbuntuOrDebianHost) { logger.debug("Running command : [dpkg -l ovmf] with timeout : " + timeout + " ms"); result = Script.executeCommandForExitValue(timeout, Script.getExecutableAbsolutePath("dpkg"), "-l", "ovmf"); } else { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtResizeVolumeCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtResizeVolumeCommandWrapper.java index 6a3901e345c..f2af46d4cc8 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtResizeVolumeCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtResizeVolumeCommandWrapper.java @@ -25,7 +25,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.Map; import com.cloud.hypervisor.kvm.storage.ScaleIOStorageAdaptor; import org.apache.cloudstack.utils.cryptsetup.KeyFile; @@ -33,7 +32,6 @@ import org.apache.cloudstack.utils.qemu.QemuImageOptions; import org.apache.cloudstack.utils.qemu.QemuImg; import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; import org.apache.cloudstack.utils.qemu.QemuImgException; -import org.apache.cloudstack.utils.qemu.QemuImgFile; import org.apache.cloudstack.utils.qemu.QemuObject; import org.libvirt.Connect; import org.libvirt.Domain; @@ -100,7 +98,7 @@ public final class LibvirtResizeVolumeCommandWrapper extends CommandWrapper info = qemu.info(qemuFile); - if (info.containsKey(QemuImg.VIRTUAL_SIZE)) { - return Long.parseLong(info.get(QemuImg.VIRTUAL_SIZE)); - } else { - throw new CloudRuntimeException("Unable to determine virtual size of volume at path " + path); - } - } catch (QemuImgException | LibvirtException ex) { - throw new CloudRuntimeException("Error when inspecting volume at path " + path, ex); - } - } - private Answer handleMultipathSCSIResize(ResizeVolumeCommand command, KVMStoragePool pool) { ((MultipathSCSIPool)pool).resize(command.getPath(), command.getInstanceName(), command.getNewSize()); return new ResizeVolumeAnswer(command, true, ""); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java index 2810f98c935..8abc359250c 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java @@ -62,16 +62,25 @@ public class LibvirtRestoreBackupCommandWrapper extends CommandWrapper(vmName, command.getVmState()), mountOptions); - } else if (Boolean.TRUE.equals(vmExists)) { - restoreVolumesOfExistingVM(volumePaths, backupPath, backupRepoType, backupRepoAddress, mountOptions); - } else { - restoreVolumesOfDestroyedVMs(volumePaths, vmName, backupPath, backupRepoType, backupRepoAddress, mountOptions); + try { + if (Objects.isNull(vmExists)) { + String volumePath = volumePaths.get(0); + int lastIndex = volumePath.lastIndexOf("/"); + newVolumeId = volumePath.substring(lastIndex + 1); + restoreVolume(backupPath, backupRepoType, backupRepoAddress, volumePath, diskType, restoreVolumeUuid, + new Pair<>(vmName, command.getVmState()), mountOptions); + } else if (Boolean.TRUE.equals(vmExists)) { + restoreVolumesOfExistingVM(volumePaths, backupPath, backupRepoType, backupRepoAddress, mountOptions); + } else { + restoreVolumesOfDestroyedVMs(volumePaths, vmName, backupPath, backupRepoType, backupRepoAddress, mountOptions); + } + } catch (CloudRuntimeException e) { + String errorMessage = "Failed to restore backup for VM: " + vmName + "."; + if (e.getMessage() != null && !e.getMessage().isEmpty()) { + errorMessage += " Details: " + e.getMessage(); + } + logger.error(errorMessage); + return new BackupAnswer(command, false, errorMessage); } return new BackupAnswer(command, true, newVolumeId); @@ -86,10 +95,8 @@ public class LibvirtRestoreBackupCommandWrapper extends CommandWrapper bkpPathAndVolUuid = getBackupPath(mountDirectory, volumePath, backupPath, diskType, null); diskType = "datadisk"; - try { - replaceVolumeWithBackup(volumePath, bkpPathAndVolUuid.first()); - } catch (IOException e) { - throw new CloudRuntimeException(String.format("Unable to revert backup for volume [%s] due to [%s].", bkpPathAndVolUuid.second(), e.getMessage()), e); + if (!replaceVolumeWithBackup(volumePath, bkpPathAndVolUuid.first())) { + throw new CloudRuntimeException(String.format("Unable to restore backup for volume [%s].", bkpPathAndVolUuid.second())); } } } finally { @@ -108,10 +115,8 @@ public class LibvirtRestoreBackupCommandWrapper extends CommandWrapper bkpPathAndVolUuid = getBackupPath(mountDirectory, volumePath, backupPath, diskType, null); diskType = "datadisk"; - try { - replaceVolumeWithBackup(volumePath, bkpPathAndVolUuid.first()); - } catch (IOException e) { - throw new CloudRuntimeException(String.format("Unable to revert backup for volume [%s] due to [%s].", bkpPathAndVolUuid.second(), e.getMessage()), e); + if (!replaceVolumeWithBackup(volumePath, bkpPathAndVolUuid.first())) { + throw new CloudRuntimeException(String.format("Unable to restore backup for volume [%s].", bkpPathAndVolUuid.second())); } } } finally { @@ -126,15 +131,13 @@ public class LibvirtRestoreBackupCommandWrapper extends CommandWrapper bkpPathAndVolUuid; try { bkpPathAndVolUuid = getBackupPath(mountDirectory, volumePath, backupPath, diskType, volumeUUID); - try { - replaceVolumeWithBackup(volumePath, bkpPathAndVolUuid.first()); - if (VirtualMachine.State.Running.equals(vmNameAndState.second())) { - if (!attachVolumeToVm(vmNameAndState.first(), volumePath)) { - throw new CloudRuntimeException(String.format("Failed to attach volume to VM: %s", vmNameAndState.first())); - } + if (!replaceVolumeWithBackup(volumePath, bkpPathAndVolUuid.first())) { + throw new CloudRuntimeException(String.format("Unable to restore backup for volume [%s].", bkpPathAndVolUuid.second())); + } + if (VirtualMachine.State.Running.equals(vmNameAndState.second())) { + if (!attachVolumeToVm(vmNameAndState.first(), volumePath)) { + throw new CloudRuntimeException(String.format("Failed to attach volume to VM: %s", vmNameAndState.first())); } - } catch (IOException e) { - throw new CloudRuntimeException(String.format("Unable to revert backup for volume [%s] due to [%s].", bkpPathAndVolUuid.second(), e.getMessage()), e); } } catch (Exception e) { throw new CloudRuntimeException("Failed to restore volume", e); @@ -194,8 +197,9 @@ public class LibvirtRestoreBackupCommandWrapper extends CommandWrapper(bkpPath, volUuid); } - private void replaceVolumeWithBackup(String volumePath, String backupPath) throws IOException { - Script.runSimpleBashScript(String.format(RSYNC_COMMAND, backupPath, volumePath)); + private boolean replaceVolumeWithBackup(String volumePath, String backupPath) { + int exitValue = Script.runSimpleBashScriptForExitValue(String.format(RSYNC_COMMAND, backupPath, volumePath)); + return exitValue == 0; } private boolean attachVolumeToVm(String vmName, String volumePath) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtSetupDirectDownloadCertificateCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtSetupDirectDownloadCertificateCommandWrapper.java index 62cca7eb209..e320baad717 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtSetupDirectDownloadCertificateCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtSetupDirectDownloadCertificateCommandWrapper.java @@ -84,7 +84,7 @@ public class LibvirtSetupDirectDownloadCertificateCommandWrapper extends Command private void importCertificate(String tempCerFilePath, String keyStoreFile, String certificateName, String privatePassword) { logger.debug("Importing certificate from temporary file to keystore"); String keyToolPath = Script.getExecutableAbsolutePath("keytool"); - int result = Script.executeCommandForExitValue(keyToolPath, "-importcert", "file", tempCerFilePath, + int result = Script.executeCommandForExitValue(keyToolPath, "-importcert", "-file", tempCerFilePath, "-keystore", keyStoreFile, "-alias", sanitizeBashCommandArgument(certificateName), "-storepass", privatePassword, "-noprompt"); if (result != 0) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMPhysicalDisk.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMPhysicalDisk.java index 9d9a6415e27..c43f5101fbe 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMPhysicalDisk.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMPhysicalDisk.java @@ -16,13 +16,21 @@ // under the License. package com.cloud.hypervisor.kvm.storage; +import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.storage.formatinspector.Qcow2Inspector; +import org.apache.cloudstack.utils.imagestore.ImageStoreUtil; +import org.apache.cloudstack.utils.qemu.QemuImg; import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; +import org.apache.cloudstack.utils.qemu.QemuImgException; +import org.apache.cloudstack.utils.qemu.QemuImgFile; import org.apache.cloudstack.utils.qemu.QemuObject; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; import org.apache.commons.lang3.StringUtils; +import org.libvirt.LibvirtException; import java.util.ArrayList; import java.util.List; +import java.util.Map; public class KVMPhysicalDisk { private String path; @@ -71,6 +79,31 @@ public class KVMPhysicalDisk { return hostIp; } + public static long getVirtualSizeFromFile(String path) { + try { + QemuImg qemu = new QemuImg(0); + QemuImgFile qemuFile = new QemuImgFile(path); + Map info = qemu.info(qemuFile); + if (info.containsKey(QemuImg.VIRTUAL_SIZE)) { + return Long.parseLong(info.get(QemuImg.VIRTUAL_SIZE)); + } else { + throw new CloudRuntimeException("Unable to determine virtual size of volume at path " + path); + } + } catch (QemuImgException | LibvirtException ex) { + throw new CloudRuntimeException("Error when inspecting volume at path " + path, ex); + } + } + + public static void checkQcow2File(String path) { + if (ImageStoreUtil.isCorrectExtension(path, "qcow2")) { + try { + Qcow2Inspector.validateQcow2File(path); + } catch (RuntimeException e) { + throw new CloudRuntimeException("The volume file at path " + path + " is not a valid QCOW2. Error: " + e.getMessage()); + } + } + } + private PhysicalDiskFormat format; private long size; private long virtualSize; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java index 60ebc9c6ce9..f273bfe9dd5 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java @@ -407,17 +407,19 @@ public class KVMStoragePoolManager { String uuid = null; String sourceHost = ""; StoragePoolType protocol = null; - final String scheme = (storageUri.getScheme() != null) ? storageUri.getScheme().toLowerCase() : ""; List acceptedSchemes = List.of("nfs", "networkfilesystem", "filesystem"); - if (acceptedSchemes.contains(scheme)) { - sourcePath = storageUri.getPath(); - sourcePath = sourcePath.replace("//", "/"); - sourceHost = storageUri.getHost(); - uuid = UUID.nameUUIDFromBytes(new String(sourceHost + sourcePath).getBytes()).toString(); - protocol = scheme.equals("filesystem") ? StoragePoolType.Filesystem: StoragePoolType.NetworkFilesystem; + if (storageUri.getScheme() == null || !acceptedSchemes.contains(storageUri.getScheme().toLowerCase())) { + throw new CloudRuntimeException("Empty or unsupported storage pool uri scheme"); } - // secondary storage registers itself through here + final String scheme = storageUri.getScheme().toLowerCase(); + sourcePath = storageUri.getPath(); + sourcePath = sourcePath.replace("//", "/"); + sourceHost = storageUri.getHost(); + uuid = UUID.nameUUIDFromBytes(new String(sourceHost + sourcePath).getBytes()).toString(); + protocol = scheme.equals("filesystem") ? StoragePoolType.Filesystem: StoragePoolType.NetworkFilesystem; + + // storage registers itself through here return createStoragePool(uuid, sourceHost, 0, sourcePath, "", protocol, null, false); } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index cb0ba9ca72a..bf02e224e9f 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -84,9 +84,8 @@ import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.builder.ToStringBuilder; import org.apache.commons.lang3.builder.ToStringStyle; -import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; - +import org.apache.logging.log4j.Logger; import org.libvirt.Connect; import org.libvirt.Domain; import org.libvirt.DomainInfo; @@ -2488,7 +2487,9 @@ public class KVMStorageProcessor implements StorageProcessor { if (template != null) { templatePath = template.getPath(); } - if (StringUtils.isEmpty(templatePath)) { + if (ImageFormat.ISO.equals(cmd.getFormat())) { + logger.debug("Skipping template validations as image format is {}", cmd.getFormat()); + } else if (StringUtils.isEmpty(templatePath)) { logger.warn("Skipped validation whether downloaded file is QCOW2 for template {}, due to downloaded template path is empty", template.getName()); } else if (!new File(templatePath).exists()) { logger.warn("Skipped validation whether downloaded file is QCOW2 for template {}, due to downloaded template path is not valid: {}", template.getName(), templatePath); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java index f3731459f89..8be5e9658fa 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java @@ -733,10 +733,9 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { @Override public KVMStoragePool createStoragePool(String name, String host, int port, String path, String userInfo, StoragePoolType type, Map details, boolean isPrimaryStorage) { - logger.info("Attempting to create storage pool " + name + " (" + type.toString() + ") in libvirt"); - - StoragePool sp = null; - Connect conn = null; + logger.info("Attempting to create storage pool {} ({}) in libvirt", name, type); + StoragePool sp; + Connect conn; try { conn = LibvirtConnection.getConnection(); } catch (LibvirtException e) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java index 335ea0d03d2..195ce6c9984 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java @@ -37,6 +37,7 @@ import org.apache.cloudstack.utils.qemu.QemuImg; import org.apache.cloudstack.utils.qemu.QemuImgException; import org.apache.cloudstack.utils.qemu.QemuImgFile; import org.apache.cloudstack.utils.qemu.QemuObject; +import org.apache.commons.collections.MapUtils; import org.apache.commons.io.filefilter.WildcardFileFilter; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; @@ -581,14 +582,23 @@ public class ScaleIOStorageAdaptor implements StorageAdaptor { } if (!ScaleIOUtil.isSDCServiceActive()) { + logger.debug("SDC service is not active on host, starting it"); if (!ScaleIOUtil.startSDCService()) { return new Ternary<>(false, null, "Couldn't start SDC service on host"); } - } else if (!ScaleIOUtil.restartSDCService()) { - return new Ternary<>(false, null, "Couldn't restart SDC service on host"); + } else { + logger.debug("SDC service is active on host, re-starting it"); + if (!ScaleIOUtil.restartSDCService()) { + return new Ternary<>(false, null, "Couldn't restart SDC service on host"); + } } - return new Ternary<>( true, getSDCDetails(details), "Prepared client successfully"); + Map sdcDetails = getSDCDetails(details); + if (MapUtils.isEmpty(sdcDetails)) { + return new Ternary<>(false, null, "Couldn't get the SDC details on the host"); + } + + return new Ternary<>( true, sdcDetails, "Prepared client successfully"); } public Pair unprepareStorageClient(Storage.StoragePoolType type, String uuid) { @@ -611,20 +621,40 @@ public class ScaleIOStorageAdaptor implements StorageAdaptor { private Map getSDCDetails(Map details) { Map sdcDetails = new HashMap(); - if (details == null || !details.containsKey(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID)) { + if (MapUtils.isEmpty(details) || !details.containsKey(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID)) { return sdcDetails; } String storageSystemId = details.get(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID); - String sdcId = ScaleIOUtil.getSdcId(storageSystemId); - if (sdcId != null) { - sdcDetails.put(ScaleIOGatewayClient.SDC_ID, sdcId); - } else { - String sdcGuId = ScaleIOUtil.getSdcGuid(); - if (sdcGuId != null) { - sdcDetails.put(ScaleIOGatewayClient.SDC_GUID, sdcGuId); - } + if (StringUtils.isEmpty(storageSystemId)) { + return sdcDetails; } + + int numberOfTries = 5; + int timeBetweenTries = 1000; // Try more frequently (every sec) and return early when SDC Id or Guid found + int attempt = 1; + do { + logger.debug("Get SDC details, attempt #{}", attempt); + String sdcId = ScaleIOUtil.getSdcId(storageSystemId); + if (sdcId != null) { + sdcDetails.put(ScaleIOGatewayClient.SDC_ID, sdcId); + return sdcDetails; + } else { + String sdcGuId = ScaleIOUtil.getSdcGuid(); + if (sdcGuId != null) { + sdcDetails.put(ScaleIOGatewayClient.SDC_GUID, sdcGuId); + return sdcDetails; + } + } + + try { + Thread.sleep(timeBetweenTries); + } catch (Exception ignore) { + } + numberOfTries--; + attempt++; + } while (numberOfTries > 0); + return sdcDetails; } diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapperTest.java index 9537469145f..b369cf25f3d 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapperTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapperTest.java @@ -22,7 +22,6 @@ import java.util.List; import java.util.UUID; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; -import org.apache.cloudstack.vm.UnmanagedInstanceTO; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -40,13 +39,10 @@ import com.cloud.agent.api.to.NfsTO; import com.cloud.agent.api.to.RemoteInstanceTO; import com.cloud.hypervisor.Hypervisor; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; -import com.cloud.hypervisor.kvm.resource.LibvirtDomainXMLParser; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef; import com.cloud.hypervisor.kvm.storage.KVMPhysicalDisk; import com.cloud.hypervisor.kvm.storage.KVMStoragePool; import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; -import com.cloud.storage.Storage; -import com.cloud.utils.Pair; import com.cloud.utils.script.Script; @RunWith(MockitoJUnitRunner.class) @@ -118,72 +114,6 @@ public class LibvirtConvertInstanceCommandWrapperTest { Assert.assertEquals(relativePath, diskDef.getDiskPath()); } - @Test - public void testMoveTemporaryDisksToDestination() { - KVMPhysicalDisk sourceDisk = Mockito.mock(KVMPhysicalDisk.class); - List disks = List.of(sourceDisk); - String destinationPoolUuid = UUID.randomUUID().toString(); - List destinationPools = List.of(destinationPoolUuid); - - KVMPhysicalDisk destDisk = Mockito.mock(KVMPhysicalDisk.class); - Mockito.when(destDisk.getPath()).thenReturn("xyz"); - Mockito.when(storagePoolManager.getStoragePool(Storage.StoragePoolType.NetworkFilesystem, destinationPoolUuid)) - .thenReturn(destinationPool); - Mockito.when(destinationPool.getType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem); - Mockito.when(storagePoolManager.copyPhysicalDisk(Mockito.eq(sourceDisk), Mockito.anyString(), Mockito.eq(destinationPool), Mockito.anyInt())) - .thenReturn(destDisk); - - List movedDisks = convertInstanceCommandWrapper.moveTemporaryDisksToDestination(disks, destinationPools, storagePoolManager); - Assert.assertEquals(1, movedDisks.size()); - Assert.assertEquals("xyz", movedDisks.get(0).getPath()); - } - - @Test - public void testGetUnmanagedInstanceDisks() { - try (MockedStatic