diff --git a/.asf.yaml b/.asf.yaml index c052077c753..43f0351bc7c 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -51,16 +51,13 @@ github: collaborators: - acs-robot - - rajujith - - GaOrtiga - - SadiJr - - winterhazel - gpordeus - hsato03 - bernardodemarco - abh1sar - FelipeM525 - lucas-a-martins + - nicoschmdt protected_branches: ~ diff --git a/api/src/main/java/com/cloud/configuration/Resource.java b/api/src/main/java/com/cloud/configuration/Resource.java index bf8fca9d905..c7bf44de76c 100644 --- a/api/src/main/java/com/cloud/configuration/Resource.java +++ b/api/src/main/java/com/cloud/configuration/Resource.java @@ -21,7 +21,7 @@ public interface Resource { short RESOURCE_UNLIMITED = -1; String UNLIMITED = "Unlimited"; - enum ResourceType { // Primary and Secondary storage are allocated_storage and not the physical storage. + enum ResourceType { // All storage type resources are allocated_storage and not the physical storage. user_vm("user_vm", 0), public_ip("public_ip", 1), volume("volume", 2), @@ -33,7 +33,11 @@ public interface Resource { cpu("cpu", 8), memory("memory", 9), primary_storage("primary_storage", 10), - secondary_storage("secondary_storage", 11); + secondary_storage("secondary_storage", 11), + backup("backup", 12), + backup_storage("backup_storage", 13), + bucket("bucket", 14), + object_storage("object_storage", 15); private String name; private int ordinal; @@ -62,6 +66,10 @@ public interface Resource { } return null; } + + public static Boolean isStorageType(ResourceType type) { + return (type == primary_storage || type == secondary_storage || type == backup_storage || type == object_storage); + } } public static class ResourceOwnerType { diff --git a/api/src/main/java/com/cloud/event/EventTypes.java b/api/src/main/java/com/cloud/event/EventTypes.java index 1a814658a03..2ba27482b99 100644 --- a/api/src/main/java/com/cloud/event/EventTypes.java +++ b/api/src/main/java/com/cloud/event/EventTypes.java @@ -787,6 +787,9 @@ public class EventTypes { public static final String EVENT_SHAREDFS_EXPUNGE = "SHAREDFS.EXPUNGE"; public static final String EVENT_SHAREDFS_RECOVER = "SHAREDFS.RECOVER"; + // Resource Limit + public static final String EVENT_RESOURCE_LIMIT_UPDATE = "RESOURCE.LIMIT.UPDATE"; + static { // TODO: need a way to force author adding event types to declare the entity details as well, with out braking diff --git a/api/src/main/java/com/cloud/exception/StorageAccessException.java b/api/src/main/java/com/cloud/exception/StorageAccessException.java index eefbcf5518a..d54d77d66f1 100644 --- a/api/src/main/java/com/cloud/exception/StorageAccessException.java +++ b/api/src/main/java/com/cloud/exception/StorageAccessException.java @@ -26,7 +26,7 @@ import com.cloud.utils.SerialVersionUID; public class StorageAccessException extends RuntimeException { private static final long serialVersionUID = SerialVersionUID.StorageAccessException; - public StorageAccessException(String message) { - super(message); + public StorageAccessException(String message, Exception causer) { + super(message, causer); } } diff --git a/api/src/main/java/com/cloud/storage/Storage.java b/api/src/main/java/com/cloud/storage/Storage.java index c997f5e1dbf..05b8b3ab7a8 100644 --- a/api/src/main/java/com/cloud/storage/Storage.java +++ b/api/src/main/java/com/cloud/storage/Storage.java @@ -16,14 +16,10 @@ // under the License. package com.cloud.storage; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; - import org.apache.commons.lang.NotImplementedException; -import org.apache.commons.lang3.StringUtils; + +import java.util.ArrayList; +import java.util.List; public class Storage { public static enum ImageFormat { @@ -139,6 +135,21 @@ public class Storage { ISODISK /* Template corresponding to a iso (non root disk) present in an OVA */ } + public enum EncryptionSupport { + /** + * Encryption not supported. + */ + Unsupported, + /** + * Will use hypervisor encryption driver (qemu -> luks) + */ + Hypervisor, + /** + * Storage pool handles encryption and just provides an encrypted volume + */ + Storage + } + /** * StoragePoolTypes carry some details about the format and capabilities of a storage pool. While not necessarily a * 1:1 with PrimaryDataStoreDriver (and for KVM agent, KVMStoragePool and StorageAdaptor) implementations, it is @@ -150,61 +161,37 @@ public class Storage { * ensure this is available on the agent side as well. This is best done by defining the StoragePoolType in a common * package available on both management server and agent plugin jars. */ - public static class StoragePoolType { - private static final Map map = new LinkedHashMap<>(); + public static enum StoragePoolType { + Filesystem(false, true, EncryptionSupport.Hypervisor), // local directory + NetworkFilesystem(true, true, EncryptionSupport.Hypervisor), // NFS + IscsiLUN(true, false, EncryptionSupport.Unsupported), // shared LUN, with a clusterfs overlay + Iscsi(true, false, EncryptionSupport.Unsupported), // for e.g., ZFS Comstar + ISO(false, false, EncryptionSupport.Unsupported), // for iso image + LVM(false, false, EncryptionSupport.Unsupported), // XenServer local LVM SR + CLVM(true, false, EncryptionSupport.Unsupported), + RBD(true, true, EncryptionSupport.Unsupported), // http://libvirt.org/storage.html#StorageBackendRBD + SharedMountPoint(true, true, EncryptionSupport.Hypervisor), + VMFS(true, true, EncryptionSupport.Unsupported), // VMware VMFS storage + PreSetup(true, true, EncryptionSupport.Unsupported), // for XenServer, Storage Pool is set up by customers. + EXT(false, true, EncryptionSupport.Unsupported), // XenServer local EXT SR + OCFS2(true, false, EncryptionSupport.Unsupported), + SMB(true, false, EncryptionSupport.Unsupported), + Gluster(true, false, EncryptionSupport.Unsupported), + PowerFlex(true, true, EncryptionSupport.Hypervisor), // Dell EMC PowerFlex/ScaleIO (formerly VxFlexOS) + ManagedNFS(true, false, EncryptionSupport.Unsupported), + Linstor(true, true, EncryptionSupport.Storage), + DatastoreCluster(true, true, EncryptionSupport.Unsupported), // for VMware, to abstract pool of clusters + StorPool(true, true, EncryptionSupport.Hypervisor), + FiberChannel(true, true, EncryptionSupport.Unsupported); // Fiber Channel Pool for KVM hypervisors is used to find the volume by WWN value (/dev/disk/by-id/wwn-) - public static final StoragePoolType Filesystem = new StoragePoolType("Filesystem", false, true, true); - public static final StoragePoolType NetworkFilesystem = new StoragePoolType("NetworkFilesystem", true, true, true); - public static final StoragePoolType IscsiLUN = new StoragePoolType("IscsiLUN", true, false, false); - public static final StoragePoolType Iscsi = new StoragePoolType("Iscsi", true, false, false); - public static final StoragePoolType ISO = new StoragePoolType("ISO", false, false, false); - public static final StoragePoolType LVM = new StoragePoolType("LVM", false, false, false); - public static final StoragePoolType CLVM = new StoragePoolType("CLVM", true, false, false); - public static final StoragePoolType RBD = new StoragePoolType("RBD", true, true, false); - public static final StoragePoolType SharedMountPoint = new StoragePoolType("SharedMountPoint", true, true, true); - public static final StoragePoolType VMFS = new StoragePoolType("VMFS", true, true, false); - public static final StoragePoolType PreSetup = new StoragePoolType("PreSetup", true, true, false); - public static final StoragePoolType EXT = new StoragePoolType("EXT", false, true, false); - public static final StoragePoolType OCFS2 = new StoragePoolType("OCFS2", true, false, false); - public static final StoragePoolType SMB = new StoragePoolType("SMB", true, false, false); - public static final StoragePoolType Gluster = new StoragePoolType("Gluster", true, false, false); - public static final StoragePoolType PowerFlex = new StoragePoolType("PowerFlex", true, true, true); - public static final StoragePoolType ManagedNFS = new StoragePoolType("ManagedNFS", true, false, false); - public static final StoragePoolType Linstor = new StoragePoolType("Linstor", true, true, false); - public static final StoragePoolType DatastoreCluster = new StoragePoolType("DatastoreCluster", true, true, false); - public static final StoragePoolType StorPool = new StoragePoolType("StorPool", true,true,true); - public static final StoragePoolType FiberChannel = new StoragePoolType("FiberChannel", true,true,false); - - - private final String name; private final boolean shared; private final boolean overProvisioning; - private final boolean encryption; + private final EncryptionSupport encryption; - /** - * New StoragePoolType, set the name to check with it in Dao (Note: Do not register it into the map of pool types). - * @param name name of the StoragePoolType. - */ - public StoragePoolType(String name) { - this.name = name; - this.shared = false; - this.overProvisioning = false; - this.encryption = false; - } - - /** - * Define a new StoragePoolType, and register it into the map of pool types known to the management server. - * @param name Simple unique name of the StoragePoolType. - * @param shared Storage pool is shared/accessible to multiple hypervisors - * @param overProvisioning Storage pool supports overProvisioning - * @param encryption Storage pool supports encrypted volumes - */ - public StoragePoolType(String name, boolean shared, boolean overProvisioning, boolean encryption) { - this.name = name; + StoragePoolType(boolean shared, boolean overProvisioning, EncryptionSupport encryption) { this.shared = shared; this.overProvisioning = overProvisioning; this.encryption = encryption; - addStoragePoolType(this); } public boolean isShared() { @@ -216,50 +203,12 @@ public class Storage { } public boolean supportsEncryption() { + return encryption == EncryptionSupport.Hypervisor || encryption == EncryptionSupport.Storage; + } + + public EncryptionSupport encryptionSupportMode() { return encryption; } - - private static void addStoragePoolType(StoragePoolType storagePoolType) { - map.putIfAbsent(storagePoolType.name, storagePoolType); - } - - public static StoragePoolType[] values() { - return map.values().toArray(StoragePoolType[]::new).clone(); - } - - public static StoragePoolType valueOf(String name) { - if (StringUtils.isBlank(name)) { - return null; - } - - StoragePoolType storage = map.get(name); - if (storage == null) { - throw new IllegalArgumentException("StoragePoolType '" + name + "' not found"); - } - return storage; - } - - @Override - public String toString() { - return name; - } - - public String name() { - return name; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - StoragePoolType that = (StoragePoolType) o; - return Objects.equals(name, that.name); - } - - @Override - public int hashCode() { - return Objects.hash(name); - } } public static List getNonSharedStoragePoolTypes() { diff --git a/api/src/main/java/com/cloud/storage/VolumeApiService.java b/api/src/main/java/com/cloud/storage/VolumeApiService.java index 6f4c7aa09e2..3a72f8bddc4 100644 --- a/api/src/main/java/com/cloud/storage/VolumeApiService.java +++ b/api/src/main/java/com/cloud/storage/VolumeApiService.java @@ -190,4 +190,6 @@ public interface VolumeApiService { boolean stateTransitTo(Volume vol, Volume.Event event) throws NoTransitionException; Pair checkAndRepairVolume(CheckAndRepairVolumeCmd cmd) throws ResourceAllocationException; + + Long getVolumePhysicalSize(Storage.ImageFormat format, String path, String chainInfo); } diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 4087d76033c..9840c747afc 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -51,12 +51,19 @@ public class ApiConstants { public static final String AVAILABLE = "available"; public static final String AVAILABLE_SUBNETS = "availablesubnets"; public static final String AVAILABLE_VIRTUAL_MACHINE_COUNT = "availablevirtualmachinecount"; + public static final String BACKUP_AVAILABLE = "backupavailable"; public static final String BACKUP_ID = "backupid"; + public static final String BACKUP_LIMIT = "backuplimit"; public static final String BACKUP_OFFERING_NAME = "backupofferingname"; public static final String BACKUP_OFFERING_ID = "backupofferingid"; + public static final String BACKUP_STORAGE_AVAILABLE = "backupstorageavailable"; + public static final String BACKUP_STORAGE_LIMIT = "backupstoragelimit"; + public static final String BACKUP_STORAGE_TOTAL = "backupstoragetotal"; + public static final String BACKUP_TOTAL = "backuptotal"; public static final String BASE64_IMAGE = "base64image"; public static final String BGP_PEERS = "bgppeers"; public static final String BGP_PEER_IDS = "bgppeerids"; + public static final String BATCH_SIZE = "batchsize"; public static final String BITS = "bits"; public static final String BOOTABLE = "bootable"; public static final String BIND_DN = "binddn"; @@ -328,6 +335,7 @@ public class ApiConstants { public static final String MANUAL_UPGRADE = "manualupgrade"; public static final String MAX = "max"; public static final String MAX_SNAPS = "maxsnaps"; + public static final String MAX_BACKUPS = "maxbackups"; public static final String MAX_CPU_NUMBER = "maxcpunumber"; public static final String MAX_MEMORY = "maxmemory"; public static final String MIN_CPU_NUMBER = "mincpunumber"; @@ -444,6 +452,7 @@ public class ApiConstants { public static final String QUALIFIERS = "qualifiers"; public static final String QUERY_FILTER = "queryfilter"; public static final String SCHEDULE = "schedule"; + public static final String SCHEDULE_ID = "scheduleid"; public static final String SCOPE = "scope"; public static final String SEARCH_BASE = "searchbase"; public static final String SECONDARY_IP = "secondaryip"; @@ -455,7 +464,6 @@ public class ApiConstants { public static final String SENT = "sent"; public static final String SENT_BYTES = "sentbytes"; public static final String SERIAL = "serial"; - public static final String SERVICE_IP = "serviceip"; public static final String SERVICE_OFFERING_ID = "serviceofferingid"; public static final String SESSIONKEY = "sessionkey"; public static final String SHOW_CAPACITIES = "showcapacities"; @@ -485,11 +493,12 @@ public class ApiConstants { public static final String STATE = "state"; public static final String STATS = "stats"; public static final String STATUS = "status"; - public static final String STORAGE_TYPE = "storagetype"; - public static final String STORAGE_POLICY = "storagepolicy"; - public static final String STORAGE_MOTION_ENABLED = "storagemotionenabled"; public static final String STORAGE_CAPABILITIES = "storagecapabilities"; public static final String STORAGE_CUSTOM_STATS = "storagecustomstats"; + public static final String STORAGE_MOTION_ENABLED = "storagemotionenabled"; + public static final String STORAGE_POLICY = "storagepolicy"; + public static final String STORAGE_POOL = "storagepool"; + public static final String STORAGE_TYPE = "storagetype"; public static final String SUBNET = "subnet"; public static final String OWNER = "owner"; public static final String SWAP_OWNER = "swapowner"; @@ -969,7 +978,6 @@ public class ApiConstants { public static final String AUTOSCALE_VMGROUP_NAME = "autoscalevmgroupname"; public static final String BAREMETAL_DISCOVER_NAME = "baremetaldiscovername"; public static final String BAREMETAL_RCT_URL = "baremetalrcturl"; - public static final String BATCH_SIZE = "batchsize"; public static final String UCS_DN = "ucsdn"; public static final String GSLB_PROVIDER = "gslbprovider"; public static final String EXCLUSIVE_GSLB_PROVIDER = "isexclusivegslbprovider"; @@ -1169,7 +1177,6 @@ public class ApiConstants { public static final String MTU = "mtu"; public static final String AUTO_ENABLE_KVM_HOST = "autoenablekvmhost"; public static final String LIST_APIS = "listApis"; - public static final String OBJECT_STORAGE_ID = "objectstorageid"; public static final String VERSIONING = "versioning"; public static final String OBJECT_LOCKING = "objectlocking"; public static final String ENCRYPTION = "encryption"; @@ -1183,7 +1190,6 @@ public class ApiConstants { public static final String DISK_PATH = "diskpath"; public static final String IMPORT_SOURCE = "importsource"; public static final String TEMP_PATH = "temppath"; - public static final String OBJECT_STORAGE = "objectstore"; public static final String HEURISTIC_RULE = "heuristicrule"; public static final String HEURISTIC_TYPE_VALID_OPTIONS = "Valid options are: ISO, SNAPSHOT, TEMPLATE and VOLUME."; public static final String MANAGEMENT = "management"; @@ -1211,6 +1217,16 @@ public class ApiConstants { public static final String SHAREDFSVM_MIN_CPU_COUNT = "sharedfsvmmincpucount"; public static final String SHAREDFSVM_MIN_RAM_SIZE = "sharedfsvmminramsize"; + // Object Storage related + public static final String BUCKET_AVAILABLE = "bucketavailable"; + public static final String BUCKET_LIMIT = "bucketlimit"; + public static final String BUCKET_TOTAL = "buckettotal"; + public static final String OBJECT_STORAGE_ID = "objectstorageid"; + public static final String OBJECT_STORAGE = "objectstore"; + public static final String OBJECT_STORAGE_AVAILABLE = "objectstorageavailable"; + public static final String OBJECT_STORAGE_LIMIT = "objectstoragelimit"; + public static final String OBJECT_STORAGE_TOTAL = "objectstoragetotal"; + public static final String PARAMETER_DESCRIPTION_ACTIVATION_RULE = "Quota tariff's activation rule. It can receive a JS script that results in either " + "a boolean or a numeric value: if it results in a boolean value, the tariff value will be applied according to the result; if it results in a numeric value, the " + "numeric value will be applied; if the result is neither a boolean nor a numeric value, the tariff will not be applied. If the rule is not informed, the tariff " + @@ -1224,6 +1240,8 @@ public class ApiConstants { "however, the following formats are also accepted: \"yyyy-MM-dd HH:mm:ss\" (e.g.: \"2023-01-01 12:00:00\") and \"yyyy-MM-dd\" (e.g.: \"2023-01-01\" - if the time is not " + "added, it will be interpreted as \"23:59:59\"). If the recommended format is not used, the date will be considered in the server timezone."; + public static final String VMWARE_DC = "vmwaredc"; + /** * This enum specifies IO Drivers, each option controls specific policies on I/O. * Qemu guests support "threads" and "native" options Since 0.8.8 ; "io_uring" is supported Since 6.3.0 (QEMU 5.0). diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupCmd.java index 558f92e4006..2d387788243 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupCmd.java @@ -19,6 +19,7 @@ package org.apache.cloudstack.api.command.user.backup; import javax.inject.Inject; +import com.cloud.storage.Snapshot; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiCommandResourceType; @@ -27,6 +28,7 @@ import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseAsyncCreateCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.BackupScheduleResponse; import org.apache.cloudstack.api.response.SuccessResponse; import org.apache.cloudstack.api.response.UserVmResponse; import org.apache.cloudstack.backup.BackupManager; @@ -60,6 +62,13 @@ public class CreateBackupCmd extends BaseAsyncCreateCmd { description = "ID of the VM") private Long vmId; + @Parameter(name = ApiConstants.SCHEDULE_ID, + type = CommandType.LONG, + entityType = BackupScheduleResponse.class, + description = "backup schedule ID of the VM, if this is null, it indicates that it is a manual backup.", + since = "4.21.0") + private Long scheduleId; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -68,6 +77,14 @@ public class CreateBackupCmd extends BaseAsyncCreateCmd { return vmId; } + public Long getScheduleId() { + if (scheduleId != null) { + return scheduleId; + } else { + return Snapshot.MANUAL_POLICY_ID; + } + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @@ -75,7 +92,7 @@ public class CreateBackupCmd extends BaseAsyncCreateCmd { @Override public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { try { - boolean result = backupManager.createBackup(getVmId()); + boolean result = backupManager.createBackup(getVmId(), getScheduleId()); if (result) { SuccessResponse response = new SuccessResponse(getCommandName()); response.setResponseName(getCommandName()); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupScheduleCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupScheduleCmd.java index 5dc06af2123..1d0741e6217 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupScheduleCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupScheduleCmd.java @@ -75,6 +75,12 @@ public class CreateBackupScheduleCmd extends BaseCmd { description = "Specifies a timezone for this command. For more information on the timezone parameter, see TimeZone Format.") private String timezone; + @Parameter(name = ApiConstants.MAX_BACKUPS, + type = CommandType.INTEGER, + description = "maximum number of backups to retain", + since = "4.21.0") + private Integer maxBackups; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -95,6 +101,10 @@ public class CreateBackupScheduleCmd extends BaseCmd { return timezone; } + public Integer getMaxBackups() { + return maxBackups; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/CreateBucketCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/CreateBucketCmd.java index d2c91e57871..722556b8e2d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/CreateBucketCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/CreateBucketCmd.java @@ -72,7 +72,7 @@ public class CreateBucketCmd extends BaseAsyncCreateCmd implements UserCmd { description = "Id of the Object Storage Pool where bucket is created") private long objectStoragePoolId; - @Parameter(name = ApiConstants.QUOTA, type = CommandType.INTEGER,description = "Bucket Quota in GB") + @Parameter(name = ApiConstants.QUOTA, type = CommandType.INTEGER, required = true, description = "Bucket Quota in GiB") private Integer quota; @Parameter(name = ApiConstants.ENCRYPTION, type = CommandType.BOOLEAN, description = "Enable bucket encryption") diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/UpdateBucketCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/UpdateBucketCmd.java index 8e281b20e91..f913373c04b 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/UpdateBucketCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/UpdateBucketCmd.java @@ -56,7 +56,7 @@ public class UpdateBucketCmd extends BaseCmd { @Parameter(name = ApiConstants.POLICY, type = CommandType.STRING, description = "Bucket Access Policy") private String policy; - @Parameter(name = ApiConstants.QUOTA, type = CommandType.INTEGER,description = "Bucket Quota in GB") + @Parameter(name = ApiConstants.QUOTA, type = CommandType.INTEGER, description = "Bucket Quota in GiB") private Integer quota; ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/response/AccountResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/AccountResponse.java index 6fc098295f6..aaad7f985fc 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/AccountResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/AccountResponse.java @@ -127,6 +127,30 @@ public class AccountResponse extends BaseResponse implements ResourceLimitAndCou @Param(description = "the total number of snapshots available for this account") private String snapshotAvailable; + @SerializedName(ApiConstants.BACKUP_LIMIT) + @Param(description = "the total number of backups which can be stored by this account", since = "4.21.0") + private String backupLimit; + + @SerializedName(ApiConstants.BACKUP_TOTAL) + @Param(description = "the total number of backups stored by this account", since = "4.21.0") + private Long backupTotal; + + @SerializedName(ApiConstants.BACKUP_AVAILABLE) + @Param(description = "the total number of backups available to this account", since = "4.21.0") + private String backupAvailable; + + @SerializedName(ApiConstants.BACKUP_STORAGE_LIMIT) + @Param(description = "the total backup storage space (in GiB) the account can own", since = "4.21.0") + private String backupStorageLimit; + + @SerializedName(ApiConstants.BACKUP_STORAGE_TOTAL) + @Param(description = "the total backup storage space (in GiB) owned by the account", since = "4.21.0") + private Long backupStorageTotal; + + @SerializedName(ApiConstants.BACKUP_STORAGE_AVAILABLE) + @Param(description = "the total backup storage space (in GiB) available to the account", since = "4.21.0") + private String backupStorageAvailable; + @SerializedName("templatelimit") @Param(description = "the total number of templates which can be created by this account") private String templateLimit; @@ -231,6 +255,30 @@ public class AccountResponse extends BaseResponse implements ResourceLimitAndCou @Param(description = "the total secondary storage space (in GiB) available to be used for this account", since = "4.2.0") private String secondaryStorageAvailable; + @SerializedName(ApiConstants.BUCKET_LIMIT) + @Param(description = "the total number of buckets which can be stored by this account", since = "4.21.0") + private String bucketLimit; + + @SerializedName(ApiConstants.BUCKET_TOTAL) + @Param(description = "the total number of buckets stored by this account", since = "4.21.0") + private Long bucketTotal; + + @SerializedName(ApiConstants.BUCKET_AVAILABLE) + @Param(description = "the total number of buckets available to this account", since = "4.21.0") + private String bucketAvailable; + + @SerializedName(ApiConstants.OBJECT_STORAGE_LIMIT) + @Param(description = "the total object storage space (in GiB) the account can own", since = "4.21.0") + private String objectStorageLimit; + + @SerializedName(ApiConstants.OBJECT_STORAGE_TOTAL) + @Param(description = "the total object storage space (in GiB) owned by the account", since = "4.21.0") + private Long objectStorageTotal; + + @SerializedName(ApiConstants.OBJECT_STORAGE_AVAILABLE) + @Param(description = "the total object storage space (in GiB) available to the account", since = "4.21.0") + private String objectStorageAvailable; + @SerializedName(ApiConstants.STATE) @Param(description = "the state of the account") private String state; @@ -386,6 +434,36 @@ public class AccountResponse extends BaseResponse implements ResourceLimitAndCou this.snapshotAvailable = snapshotAvailable; } + @Override + public void setBackupLimit(String backupLimit) { + this.backupLimit = backupLimit; + } + + @Override + public void setBackupTotal(Long backupTotal) { + this.backupTotal = backupTotal; + } + + @Override + public void setBackupAvailable(String backupAvailable) { + this.backupAvailable = backupAvailable; + } + + @Override + public void setBackupStorageLimit(String backupStorageLimit) { + this.backupStorageLimit = backupStorageLimit; + } + + @Override + public void setBackupStorageTotal(Long backupStorageTotal) { + this.backupStorageTotal = backupStorageTotal; + } + + @Override + public void setBackupStorageAvailable(String backupStorageAvailable) { + this.backupStorageAvailable = backupStorageAvailable; + } + @Override public void setTemplateLimit(String templateLimit) { this.templateLimit = templateLimit; @@ -537,6 +615,36 @@ public class AccountResponse extends BaseResponse implements ResourceLimitAndCou this.secondaryStorageAvailable = secondaryStorageAvailable; } + @Override + public void setBucketLimit(String bucketLimit) { + this.bucketLimit = bucketLimit; + } + + @Override + public void setBucketTotal(Long bucketTotal) { + this.bucketTotal = bucketTotal; + } + + @Override + public void setBucketAvailable(String bucketAvailable) { + this.bucketAvailable = bucketAvailable; + } + + @Override + public void setObjectStorageLimit(String objectStorageLimit) { + this.objectStorageLimit = objectStorageLimit; + } + + @Override + public void setObjectStorageTotal(Long objectStorageTotal) { + this.objectStorageTotal = objectStorageTotal; + } + + @Override + public void setObjectStorageAvailable(String objectStorageAvailable) { + this.objectStorageAvailable = objectStorageAvailable; + } + public void setDefaultZone(String defaultZoneId) { this.defaultZoneId = defaultZoneId; } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/BackupScheduleResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/BackupScheduleResponse.java index ba44f1e024f..d7c6f96add5 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/BackupScheduleResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/BackupScheduleResponse.java @@ -37,18 +37,22 @@ public class BackupScheduleResponse extends BaseResponse { @Param(description = "ID of the VM") private String vmId; - @SerializedName("schedule") + @SerializedName(ApiConstants.SCHEDULE) @Param(description = "time the backup is scheduled to be taken.") private String schedule; - @SerializedName("intervaltype") + @SerializedName(ApiConstants.INTERVAL_TYPE) @Param(description = "the interval type of the backup schedule") private DateUtil.IntervalType intervalType; - @SerializedName("timezone") + @SerializedName(ApiConstants.TIMEZONE) @Param(description = "the time zone of the backup schedule") private String timezone; + @SerializedName(ApiConstants.MAX_BACKUPS) + @Param(description = "maximum number of backups retained") + private Integer maxBakups; + public String getVmName() { return vmName; } @@ -88,4 +92,8 @@ public class BackupScheduleResponse extends BaseResponse { public void setTimezone(String timezone) { this.timezone = timezone; } + + public void setMaxBakups(Integer maxBakups) { + this.maxBakups = maxBakups; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/BucketResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/BucketResponse.java index f2dd365452c..cde140839ec 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/BucketResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/BucketResponse.java @@ -75,7 +75,7 @@ public class BucketResponse extends BaseResponseWithTagInformation implements Co private String state; @SerializedName(ApiConstants.QUOTA) - @Param(description = "Bucket Quota in GB") + @Param(description = "Bucket Quota in GiB") private Integer quota; @SerializedName(ApiConstants.ENCRYPTION) diff --git a/api/src/main/java/org/apache/cloudstack/api/response/DomainResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/DomainResponse.java index 7c6ad3a91c3..74fa2cbb1e4 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/DomainResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/DomainResponse.java @@ -105,6 +105,30 @@ public class DomainResponse extends BaseResponseWithAnnotations implements Resou @SerializedName("snapshotavailable") @Param(description="the total number of snapshots available for this domain") private String snapshotAvailable; + @SerializedName(ApiConstants.BACKUP_LIMIT) + @Param(description = "the total number of backups which can be stored by this domain", since = "4.21.0") + private String backupLimit; + + @SerializedName(ApiConstants.BACKUP_TOTAL) + @Param(description = "the total number of backups stored by this domain", since = "4.21.0") + private Long backupTotal; + + @SerializedName(ApiConstants.BACKUP_AVAILABLE) + @Param(description = "the total number of backups available to this domain", since = "4.21.0") + private String backupAvailable; + + @SerializedName(ApiConstants.BACKUP_STORAGE_LIMIT) + @Param(description = "the total backup storage space (in GiB) the domain can own", since = "4.21.0") + private String backupStorageLimit; + + @SerializedName(ApiConstants.BACKUP_STORAGE_TOTAL) + @Param(description = "the total backup storage space (in GiB) owned by the domain", since = "4.21.0") + private Long backupStorageTotal; + + @SerializedName(ApiConstants.BACKUP_STORAGE_AVAILABLE) + @Param(description = "the total backup storage space (in GiB) available to the domain", since = "4.21.0") + private String backupStorageAvailable; + @SerializedName("templatelimit") @Param(description="the total number of templates which can be created by this domain") private String templateLimit; @@ -177,6 +201,30 @@ public class DomainResponse extends BaseResponseWithAnnotations implements Resou @SerializedName("secondarystorageavailable") @Param(description="the total secondary storage space (in GiB) available to be used for this domain", since="4.2.0") private String secondaryStorageAvailable; + @SerializedName(ApiConstants.BUCKET_LIMIT) + @Param(description = "the total number of buckets which can be stored by this domain", since = "4.21.0") + private String bucketLimit; + + @SerializedName(ApiConstants.BUCKET_TOTAL) + @Param(description = "the total number of buckets stored by this domain", since = "4.21.0") + private Long bucketTotal; + + @SerializedName(ApiConstants.BUCKET_AVAILABLE) + @Param(description = "the total number of buckets available to this domain", since = "4.21.0") + private String bucketAvailable; + + @SerializedName(ApiConstants.OBJECT_STORAGE_LIMIT) + @Param(description = "the total object storage space (in GiB) the domain can own", since = "4.21.0") + private String objectStorageLimit; + + @SerializedName(ApiConstants.OBJECT_STORAGE_TOTAL) + @Param(description = "the total object storage space (in GiB) owned by the domain", since = "4.21.0") + private Long objectStorageTotal; + + @SerializedName(ApiConstants.OBJECT_STORAGE_AVAILABLE) + @Param(description = "the total object storage space (in GiB) available to the domain", since = "4.21.0") + private String objectStorageAvailable; + @SerializedName(ApiConstants.RESOURCE_ICON) @Param(description = "Base64 string representation of the resource icon", since = "4.16.0.0") ResourceIconResponse icon; @@ -313,6 +361,36 @@ public class DomainResponse extends BaseResponseWithAnnotations implements Resou this.snapshotAvailable = snapshotAvailable; } + @Override + public void setBackupLimit(String backupLimit) { + this.backupLimit = backupLimit; + } + + @Override + public void setBackupTotal(Long backupTotal) { + this.backupTotal = backupTotal; + } + + @Override + public void setBackupAvailable(String backupAvailable) { + this.backupAvailable = backupAvailable; + } + + @Override + public void setBackupStorageLimit(String backupStorageLimit) { + this.backupStorageLimit = backupStorageLimit; + } + + @Override + public void setBackupStorageTotal(Long backupStorageTotal) { + this.backupStorageTotal = backupStorageTotal; + } + + @Override + public void setBackupStorageAvailable(String backupStorageAvailable) { + this.backupStorageAvailable = backupStorageAvailable; + } + @Override public void setTemplateLimit(String templateLimit) { this.templateLimit = templateLimit; @@ -430,6 +508,36 @@ public class DomainResponse extends BaseResponseWithAnnotations implements Resou this.secondaryStorageAvailable = secondaryStorageAvailable; } + @Override + public void setBucketLimit(String bucketLimit) { + this.bucketLimit = bucketLimit; + } + + @Override + public void setBucketTotal(Long bucketTotal) { + this.bucketTotal = bucketTotal; + } + + @Override + public void setBucketAvailable(String bucketAvailable) { + this.bucketAvailable = bucketAvailable; + } + + @Override + public void setObjectStorageLimit(String objectStorageLimit) { + this.objectStorageLimit = objectStorageLimit; + } + + @Override + public void setObjectStorageTotal(Long objectStorageTotal) { + this.objectStorageTotal = objectStorageTotal; + } + + @Override + public void setObjectStorageAvailable(String objectStorageAvailable) { + this.objectStorageAvailable = objectStorageAvailable; + } + public void setState(String state) { this.state = state; } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java index 091d6391b31..c9a5c47887d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java @@ -152,7 +152,7 @@ public class HostResponse extends BaseResponseWithAnnotations { @Deprecated @SerializedName("memoryallocated") @Param(description = "the amount of the host's memory currently allocated") - private long memoryAllocated; + private Long memoryAllocated; @SerializedName("memoryallocatedpercentage") @Param(description = "the amount of the host's memory currently allocated in percentage") @@ -415,7 +415,7 @@ public class HostResponse extends BaseResponseWithAnnotations { this.memWithOverprovisioning=memWithOverprovisioning; } - public void setMemoryAllocated(long memoryAllocated) { + public void setMemoryAllocated(Long memoryAllocated) { this.memoryAllocated = memoryAllocated; } @@ -703,8 +703,8 @@ public class HostResponse extends BaseResponseWithAnnotations { return memoryTotal; } - public long getMemoryAllocated() { - return memoryAllocated; + public Long getMemoryAllocated() { + return memoryAllocated == null ? 0 : memoryAllocated; } public void setMemoryAllocatedPercentage(String memoryAllocatedPercentage) { diff --git a/api/src/main/java/org/apache/cloudstack/api/response/ManagementServerResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/ManagementServerResponse.java index df55a63a060..729fb5ff3bc 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/ManagementServerResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/ManagementServerResponse.java @@ -74,9 +74,9 @@ public class ManagementServerResponse extends BaseResponse { @Param(description = "the running OS kernel version for this Management Server") private String kernelVersion; - @SerializedName(ApiConstants.SERVICE_IP) + @SerializedName(ApiConstants.IP_ADDRESS) @Param(description = "the IP Address for this Management Server") - private String serviceIp; + private String ipAddress; @SerializedName(ApiConstants.PEERS) @Param(description = "the Management Server Peers") @@ -130,8 +130,8 @@ public class ManagementServerResponse extends BaseResponse { return lastBoot; } - public String getServiceIp() { - return serviceIp; + public String getIpAddress() { + return ipAddress; } public Long getAgentsCount() { @@ -186,8 +186,8 @@ public class ManagementServerResponse extends BaseResponse { this.kernelVersion = kernelVersion; } - public void setServiceIp(String serviceIp) { - this.serviceIp = serviceIp; + public void setIpAddress(String ipAddress) { + this.ipAddress = ipAddress; } public void setAgentsCount(Long agentsCount) { diff --git a/api/src/main/java/org/apache/cloudstack/api/response/ProjectResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/ProjectResponse.java index 1c63697559b..8bdf042add0 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/ProjectResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/ProjectResponse.java @@ -140,6 +140,30 @@ public class ProjectResponse extends BaseResponse implements ResourceLimitAndCou @Param(description = "the total secondary storage space (in GiB) available to be used for this project", since = "4.2.0") private String secondaryStorageAvailable; + @SerializedName(ApiConstants.BUCKET_LIMIT) + @Param(description = "the total number of buckets which can be stored by this project", since = "4.21.0") + private String bucketLimit; + + @SerializedName(ApiConstants.BUCKET_TOTAL) + @Param(description = "the total number of buckets stored by this project", since = "4.21.0") + private Long bucketTotal; + + @SerializedName(ApiConstants.BUCKET_AVAILABLE) + @Param(description = "the total number of buckets available to this project", since = "4.21.0") + private String bucketAvailable; + + @SerializedName(ApiConstants.OBJECT_STORAGE_LIMIT) + @Param(description = "the total object storage space (in GiB) the project can own", since = "4.21.0") + private String objectStorageLimit; + + @SerializedName(ApiConstants.OBJECT_STORAGE_TOTAL) + @Param(description = "the total object storage space (in GiB) owned by the project", since = "4.21.0") + private Long objectStorageTotal; + + @SerializedName(ApiConstants.OBJECT_STORAGE_AVAILABLE) + @Param(description = "the total object storage space (in GiB) available to the project", since = "4.21.0") + private String objectStorageAvailable; + @SerializedName(ApiConstants.VM_LIMIT) @Param(description = "the total number of virtual machines that can be deployed by this project", since = "4.2.0") private String vmLimit; @@ -188,6 +212,30 @@ public class ProjectResponse extends BaseResponse implements ResourceLimitAndCou @Param(description = "the total number of snapshots available for this project", since = "4.2.0") private String snapshotAvailable; + @SerializedName(ApiConstants.BACKUP_LIMIT) + @Param(description = "the total number of backups which can be stored by this project", since = "4.21.0") + private String backupLimit; + + @SerializedName(ApiConstants.BACKUP_TOTAL) + @Param(description = "the total number of backups stored by this project", since = "4.21.0") + private Long backupTotal; + + @SerializedName(ApiConstants.BACKUP_AVAILABLE) + @Param(description = "the total number of backups available to this project", since = "4.21.0") + private String backupAvailable; + + @SerializedName(ApiConstants.BACKUP_STORAGE_LIMIT) + @Param(description = "the total backup storage space (in GiB) the project can own", since = "4.21.0") + private String backupStorageLimit; + + @SerializedName(ApiConstants.BACKUP_STORAGE_TOTAL) + @Param(description = "the total backup storage space (in GiB) owned by the project", since = "4.21.0") + private Long backupStorageTotal; + + @SerializedName(ApiConstants.BACKUP_STORAGE_AVAILABLE) + @Param(description = "the total backup storage space (in GiB) available to the project", since = "4.21.0") + private String backupStorageAvailable; + @SerializedName("templatelimit") @Param(description = "the total number of templates which can be created by this project", since = "4.2.0") private String templateLimit; @@ -320,6 +368,36 @@ public class ProjectResponse extends BaseResponse implements ResourceLimitAndCou this.snapshotAvailable = snapshotAvailable; } + @Override + public void setBackupLimit(String backupLimit) { + this.backupLimit = backupLimit; + } + + @Override + public void setBackupTotal(Long backupTotal) { + this.backupTotal = backupTotal; + } + + @Override + public void setBackupAvailable(String backupAvailable) { + this.backupAvailable = backupAvailable; + } + + @Override + public void setBackupStorageLimit(String backupStorageLimit) { + this.backupStorageLimit = backupStorageLimit; + } + + @Override + public void setBackupStorageTotal(Long backupStorageTotal) { + this.backupStorageTotal = backupStorageTotal; + } + + @Override + public void setBackupStorageAvailable(String backupStorageAvailable) { + this.backupStorageAvailable = backupStorageAvailable; + } + @Override public void setTemplateLimit(String templateLimit) { this.templateLimit = templateLimit; @@ -435,6 +513,36 @@ public class ProjectResponse extends BaseResponse implements ResourceLimitAndCou this.secondaryStorageAvailable = secondaryStorageAvailable; } + @Override + public void setBucketLimit(String bucketLimit) { + this.bucketLimit = bucketLimit; + } + + @Override + public void setBucketTotal(Long bucketTotal) { + this.bucketTotal = bucketTotal; + } + + @Override + public void setBucketAvailable(String bucketAvailable) { + this.bucketAvailable = bucketAvailable; + } + + @Override + public void setObjectStorageLimit(String objectStorageLimit) { + this.objectStorageLimit = objectStorageLimit; + } + + @Override + public void setObjectStorageTotal(Long objectStorageTotal) { + this.objectStorageTotal = objectStorageTotal; + } + + @Override + public void setObjectStorageAvailable(String objectStorageAvailable) { + this.objectStorageAvailable = objectStorageAvailable; + } + public void setOwners(List> owners) { this.owners = owners; } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/ResourceLimitAndCountResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/ResourceLimitAndCountResponse.java index f9e6df3a038..b86723b36c4 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/ResourceLimitAndCountResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/ResourceLimitAndCountResponse.java @@ -84,6 +84,30 @@ public interface ResourceLimitAndCountResponse { public void setSnapshotAvailable(String snapshotAvailable); + public void setBackupLimit(String backupLimit); + + public void setBackupTotal(Long backupTotal); + + public void setBackupAvailable(String backupAvailable); + + public void setBackupStorageLimit(String backupStorageLimit); + + public void setBackupStorageTotal(Long backupStorageTotal); + + public void setBackupStorageAvailable(String backupStorageAvailable); + + void setBucketLimit(String bucketLimit); + + void setBucketTotal(Long bucketTotal); + + void setBucketAvailable(String bucketAvailable); + + void setObjectStorageLimit(String objectStorageLimit); + + void setObjectStorageTotal(Long objectStorageTotal); + + void setObjectStorageAvailable(String objectStorageAvailable); + public void setTemplateLimit(String templateLimit); public void setTemplateTotal(Long templateTotal); diff --git a/api/src/main/java/org/apache/cloudstack/backup/Backup.java b/api/src/main/java/org/apache/cloudstack/backup/Backup.java index f21f20adb33..dffe8a03213 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/Backup.java +++ b/api/src/main/java/org/apache/cloudstack/backup/Backup.java @@ -33,6 +33,28 @@ public interface Backup extends ControlledEntity, InternalIdentity, Identity { Allocated, Queued, BackingUp, BackedUp, Error, Failed, Restoring, Removed, Expunged } + public enum Type { + MANUAL, HOURLY, DAILY, WEEKLY, MONTHLY; + private int max = 8; + + public void setMax(int max) { + this.max = max; + } + + public int getMax() { + return max; + } + + @Override + public String toString() { + return this.name(); + } + + public boolean equals(String snapshotType) { + return this.toString().equalsIgnoreCase(snapshotType); + } + } + class Metric { private Long backupSize = 0L; private Long dataSize = 0L; diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java index 8b45bb4ee5e..cbd4b7e0596 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java @@ -19,6 +19,7 @@ package org.apache.cloudstack.backup; import java.util.List; +import com.cloud.exception.ResourceAllocationException; import org.apache.cloudstack.api.command.admin.backup.ImportBackupOfferingCmd; import org.apache.cloudstack.api.command.admin.backup.UpdateBackupOfferingCmd; import org.apache.cloudstack.api.command.user.backup.CreateBackupScheduleCmd; @@ -56,6 +57,86 @@ public interface BackupManager extends BackupService, Configurable, PluggableSer "false", "Enable volume attach/detach operations for VMs that are assigned to Backup Offerings.", true); + ConfigKey BackupHourlyMax = new ConfigKey("Advanced", Integer.class, + "backup.max.hourly", + "8", + "Maximum recurring hourly backups to be retained for an instance. If the limit is reached, early backups from the start of the hour are deleted so that newer ones can be saved. This limit does not apply to manual backups. If set to 0, recurring hourly backups can not be scheduled.", + false, + ConfigKey.Scope.Global, + null); + + ConfigKey BackupDailyMax = new ConfigKey("Advanced", Integer.class, + "backup.max.daily", + "8", + "Maximum recurring daily backups to be retained for an instance. If the limit is reached, backups from the start of the day are deleted so that newer ones can be saved. This limit does not apply to manual backups. If set to 0, recurring daily backups can not be scheduled.", + false, + ConfigKey.Scope.Global, + null); + + ConfigKey BackupWeeklyMax = new ConfigKey("Advanced", Integer.class, + "backup.max.weekly", + "8", + "Maximum recurring weekly backups to be retained for an instance. If the limit is reached, backups from the beginning of the week are deleted so that newer ones can be saved. This limit does not apply to manual backups. If set to 0, recurring weekly backups can not be scheduled.", + false, + ConfigKey.Scope.Global, + null); + + ConfigKey BackupMonthlyMax = new ConfigKey("Advanced", Integer.class, + "backup.max.monthly", + "8", + "Maximum recurring monthly backups to be retained for an instance. If the limit is reached, backups from the beginning of the month are deleted so that newer ones can be saved. This limit does not apply to manual backups. If set to 0, recurring monthly backups can not be scheduled.", + false, + ConfigKey.Scope.Global, + null); + + ConfigKey DefaultMaxAccountBackups = new ConfigKey("Account Defaults", Long.class, + "max.account.backups", + "20", + "The default maximum number of backups that can be created for an account", + false, + ConfigKey.Scope.Global, + null); + + ConfigKey DefaultMaxAccountBackupStorage = new ConfigKey("Account Defaults", Long.class, + "max.account.backup.storage", + "400", + "The default maximum backup storage space (in GiB) that can be used for an account", + false, + ConfigKey.Scope.Global, + null); + + ConfigKey DefaultMaxProjectBackups = new ConfigKey("Project Defaults", Long.class, + "max.project.backups", + "20", + "The default maximum number of backups that can be created for a project", + false, + ConfigKey.Scope.Global, + null); + + ConfigKey DefaultMaxProjectBackupStorage = new ConfigKey("Project Defaults", Long.class, + "max.project.backup.storage", + "400", + "The default maximum backup storage space (in GiB) that can be used for a project", + false, + ConfigKey.Scope.Global, + null); + + ConfigKey DefaultMaxDomainBackups = new ConfigKey("Domain Defaults", Long.class, + "max.domain.backups", + "40", + "The default maximum number of backups that can be created for a domain", + false, + ConfigKey.Scope.Global, + null); + + ConfigKey DefaultMaxDomainBackupStorage = new ConfigKey("Domain Defaults", Long.class, + "max.domain.backup.storage", + "800", + "The default maximum backup storage space (in GiB) that can be used for a domain", + false, + ConfigKey.Scope.Global, + null); + /** * List backup provider offerings * @param zoneId zone id @@ -119,9 +200,10 @@ public interface BackupManager extends BackupService, Configurable, PluggableSer /** * Creates backup of a VM * @param vmId Virtual Machine ID + * @param scheduleId Virtual Machine Backup Schedule ID * @return returns operation success */ - boolean createBackup(final Long vmId); + boolean createBackup(final Long vmId, final Long scheduleId) throws ResourceAllocationException; /** * List existing backups for a VM diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java b/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java index d36dfb7360f..e3a6c3a62bd 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java @@ -75,7 +75,7 @@ public interface BackupProvider { * @param backup * @return */ - boolean takeBackup(VirtualMachine vm); + Pair takeBackup(VirtualMachine vm); /** * Delete an existing backup @@ -104,9 +104,16 @@ public interface BackupProvider { Map getBackupMetrics(Long zoneId, List vms); /** - * This method should reconcile and create backup entries for any backups created out-of-band - * @param vm + * This method should TODO + * @param + */ + public List listRestorePoints(VirtualMachine vm); + + /** + * This method should TODO + * @param + * @param * @param metric */ - void syncBackups(VirtualMachine vm, Backup.Metric metric); + Backup createNewBackupEntryForRestorePoint(Backup.RestorePoint restorePoint, VirtualMachine vm, Backup.Metric metric); } diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupSchedule.java b/api/src/main/java/org/apache/cloudstack/backup/BackupSchedule.java index d81dd731b1f..4ff946be9cd 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupSchedule.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupSchedule.java @@ -30,4 +30,5 @@ public interface BackupSchedule extends InternalIdentity { String getTimezone(); Date getScheduledTimestamp(); Long getAsyncJobId(); + Integer getMaxBackups(); } diff --git a/api/src/main/java/org/apache/cloudstack/storage/object/BucketApiService.java b/api/src/main/java/org/apache/cloudstack/storage/object/BucketApiService.java index 7e1361d1e71..e27ef308d7f 100644 --- a/api/src/main/java/org/apache/cloudstack/storage/object/BucketApiService.java +++ b/api/src/main/java/org/apache/cloudstack/storage/object/BucketApiService.java @@ -22,10 +22,59 @@ import com.cloud.exception.ResourceAllocationException; import com.cloud.user.Account; import org.apache.cloudstack.api.command.user.bucket.CreateBucketCmd; import org.apache.cloudstack.api.command.user.bucket.UpdateBucketCmd; +import org.apache.cloudstack.framework.config.ConfigKey; public interface BucketApiService { + ConfigKey DefaultMaxAccountBuckets = new ConfigKey("Account Defaults", Long.class, + "max.account.buckets", + "20", + "The default maximum number of buckets that can be created for an account", + false, + ConfigKey.Scope.Global, + null); + + ConfigKey DefaultMaxAccountObjectStorage = new ConfigKey("Account Defaults", Long.class, + "max.account.object.storage", + "400", + "The default maximum object storage space (in GiB) that can be used for an account", + false, + ConfigKey.Scope.Global, + null); + + ConfigKey DefaultMaxProjectBuckets = new ConfigKey("Project Defaults", Long.class, + "max.project.buckets", + "20", + "The default maximum number of buckets that can be created for a project", + false, + ConfigKey.Scope.Global, + null); + + ConfigKey DefaultMaxProjectObjectStorage = new ConfigKey("Project Defaults", Long.class, + "max.project.object.storage", + "400", + "The default maximum object storage space (in GiB) that can be used for a project", + false, + ConfigKey.Scope.Global, + null); + + ConfigKey DefaultMaxDomainBuckets = new ConfigKey("Domain Defaults", Long.class, + "max.domain.buckets", + "20", + "The default maximum number of buckets that can be created for a domain", + false, + ConfigKey.Scope.Global, + null); + + ConfigKey DefaultMaxDomainObjectStorage = new ConfigKey("Domain Defaults", Long.class, + "max.domain.object.storage", + "400", + "The default maximum object storage space (in GiB) that can be used for a domain", + false, + ConfigKey.Scope.Global, + null); + /** * Creates the database object for a Bucket based on the given criteria * @@ -48,7 +97,7 @@ public interface BucketApiService { boolean deleteBucket(long bucketId, Account caller); - boolean updateBucket(UpdateBucketCmd cmd, Account caller); + boolean updateBucket(UpdateBucketCmd cmd, Account caller) throws ResourceAllocationException; void getBucketUsage(); } diff --git a/core/src/main/java/com/cloud/agent/api/CleanupVMCommand.java b/core/src/main/java/com/cloud/agent/api/CleanupVMCommand.java new file mode 100644 index 00000000000..a4d73a8b164 --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/CleanupVMCommand.java @@ -0,0 +1,46 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.agent.api; + +/** + * This command will destroy a leftover VM during the expunge process if it wasn't destroyed before. + * + */ +public class CleanupVMCommand extends Command { + String vmName; + boolean executeInSequence; + + public CleanupVMCommand(String vmName) { + this(vmName, false); + } + public CleanupVMCommand(String vmName, boolean executeInSequence) { + this.vmName = vmName; + this.executeInSequence = executeInSequence; + } + + @Override + public boolean executeInSequence() { + return executeInSequence; + } + + public String getVmName() { + return vmName; + } +} diff --git a/debian/control b/debian/control index a773844c27c..1292639ef30 100644 --- a/debian/control +++ b/debian/control @@ -1,7 +1,7 @@ Source: cloudstack Section: libs Priority: extra -Maintainer: Wido den Hollander +Maintainer: The Apache CloudStack Team Build-Depends: debhelper (>= 9), openjdk-17-jdk | java17-sdk | java17-jdk | zulu-17 | openjdk-11-jdk | java11-sdk | java11-jdk | zulu-11, genisoimage, python-mysql.connector | python3-mysql.connector | mysql-connector-python-py3, maven (>= 3) | maven3, python (>= 2.7) | python2 (>= 2.7), python3 (>= 3), python-setuptools, python3-setuptools, diff --git a/engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java b/engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java index c3d45b98b00..4c81c7359f2 100644 --- a/engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java +++ b/engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java @@ -16,6 +16,8 @@ // under the License. package com.cloud.capacity; +import java.util.List; + import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; @@ -67,7 +69,7 @@ public interface CapacityManager { "0.85", "Percentage (as a value between 0 and 1) of storage utilization above which allocators will disable using the pool for low storage available.", true, - ConfigKey.Scope.Zone); + List.of(ConfigKey.Scope.StoragePool, ConfigKey.Scope.Zone)); static final ConfigKey StorageOverprovisioningFactor = new ConfigKey<>( "Storage", @@ -85,7 +87,7 @@ public interface CapacityManager { "0.85", "Percentage (as a value between 0 and 1) of allocated storage utilization above which allocators will disable using the pool for low allocated storage available.", true, - ConfigKey.Scope.Zone); + List.of(ConfigKey.Scope.StoragePool, ConfigKey.Scope.Zone)); static final ConfigKey StorageOperationsExcludeCluster = new ConfigKey<>( Boolean.class, @@ -125,7 +127,7 @@ public interface CapacityManager { "Percentage (as a value between 0 and 1) of allocated storage utilization above which allocators will disable using the pool for volume resize. " + "This is applicable only when volume.resize.allowed.beyond.allocation is set to true.", true, - ConfigKey.Scope.Zone); + List.of(ConfigKey.Scope.StoragePool, ConfigKey.Scope.Zone)); ConfigKey CapacityCalculateWorkers = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, Integer.class, "capacity.calculate.workers", "1", diff --git a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java index 7b31ec6a81b..46f796b4f78 100644 --- a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java +++ b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java @@ -214,7 +214,7 @@ public interface StorageManager extends StorageService { ConfigKey AllowVolumeReSizeBeyondAllocation = new ConfigKey("Advanced", Boolean.class, "volume.resize.allowed.beyond.allocation", "false", "Determines whether volume size can exceed the pool capacity allocation disable threshold (pool.storage.allocated.capacity.disablethreshold) " + "when resize a volume upto resize capacity disable threshold (pool.storage.allocated.resize.capacity.disablethreshold)", - true, ConfigKey.Scope.Zone); + true, List.of(ConfigKey.Scope.StoragePool, ConfigKey.Scope.Zone)); ConfigKey StoragePoolHostConnectWorkers = new ConfigKey<>("Storage", Integer.class, "storage.pool.host.connect.workers", "1", diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java index 06061908888..db0119febde 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java @@ -1807,7 +1807,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati try { volService.grantAccess(volFactory.getVolume(newVol.getId()), host, destPool); } catch (Exception e) { - throw new StorageAccessException(String.format("Unable to grant access to the volume [%s] on host [%s].", newVolToString, host)); + throw new StorageAccessException(String.format("Unable to grant access to the volume [%s] on host [%s].", newVolToString, host), e); } } @@ -1847,7 +1847,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati try { volService.grantAccess(volFactory.getVolume(volumeId), host, volumeStore); } catch (Exception e) { - throw new StorageAccessException(String.format("Unable to grant access to volume [%s] on host [%s].", volToString, host)); + throw new StorageAccessException(String.format("Unable to grant access to volume [%s] on host [%s].", volToString, host), e); } } @@ -1928,7 +1928,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati try { volService.grantAccess(volFactory.getVolume(vol.getId()), host, store); } catch (Exception e) { - throw new StorageAccessException(String.format("Unable to grant access to volume [%s] on host [%s].", volToString, host)); + throw new StorageAccessException(String.format("Unable to grant access to volume [%s] on host [%s].", volToString, host), e); } } else { grantVolumeAccessToHostIfNeeded(store, vol.getId(), host, volToString); diff --git a/engine/schema/src/main/java/com/cloud/dc/ClusterDetailsDao.java b/engine/schema/src/main/java/com/cloud/dc/ClusterDetailsDao.java index 5daab544b21..27cea8d5c2d 100644 --- a/engine/schema/src/main/java/com/cloud/dc/ClusterDetailsDao.java +++ b/engine/schema/src/main/java/com/cloud/dc/ClusterDetailsDao.java @@ -20,8 +20,9 @@ import java.util.Collection; import java.util.Map; import com.cloud.utils.db.GenericDao; +import org.apache.cloudstack.resourcedetail.ResourceDetailsDao; -public interface ClusterDetailsDao extends GenericDao { +public interface ClusterDetailsDao extends GenericDao, ResourceDetailsDao { Map findDetails(long clusterId); void persist(long clusterId, Map details); diff --git a/engine/schema/src/main/java/com/cloud/dc/ClusterDetailsDaoImpl.java b/engine/schema/src/main/java/com/cloud/dc/ClusterDetailsDaoImpl.java index a4f6acb9057..4c752ff9b4f 100644 --- a/engine/schema/src/main/java/com/cloud/dc/ClusterDetailsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/dc/ClusterDetailsDaoImpl.java @@ -22,18 +22,27 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import javax.inject.Inject; + import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.ConfigKey.Scope; import org.apache.cloudstack.framework.config.ScopedConfigStorage; import org.apache.commons.collections.CollectionUtils; +import com.cloud.dc.dao.ClusterDao; +import com.cloud.org.Cluster; +import com.cloud.utils.Pair; import com.cloud.utils.crypt.DBEncryptionUtil; -import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.TransactionLegacy; +import org.apache.cloudstack.resourcedetail.ResourceDetailsDaoBase; + +public class ClusterDetailsDaoImpl extends ResourceDetailsDaoBase implements ClusterDetailsDao, ScopedConfigStorage { + + @Inject + ClusterDao clusterDao; -public class ClusterDetailsDaoImpl extends GenericDaoBase implements ClusterDetailsDao, ScopedConfigStorage { protected final SearchBuilder ClusterSearch; protected final SearchBuilder DetailSearch; @@ -44,11 +53,11 @@ public class ClusterDetailsDaoImpl extends GenericDaoBase findDetails(long clusterId) { SearchCriteria sc = ClusterSearch.create(); @@ -91,7 +105,7 @@ public class ClusterDetailsDaoImpl extends GenericDaoBase(); } SearchBuilder sb = createSearchBuilder(); - sb.and("clusterId", sb.entity().getClusterId(), SearchCriteria.Op.EQ); + sb.and("clusterId", sb.entity().getResourceId(), SearchCriteria.Op.EQ); sb.and("name", sb.entity().getName(), SearchCriteria.Op.IN); sb.done(); SearchCriteria sc = sb.create(); @@ -180,4 +194,13 @@ public class ClusterDetailsDaoImpl extends GenericDaoBase getParentScope(long id) { + Cluster cluster = clusterDao.findById(id); + if (cluster == null) { + return null; + } + return new Pair<>(getScope().getParent(), cluster.getDataCenterId()); + } } diff --git a/engine/schema/src/main/java/com/cloud/dc/ClusterDetailsVO.java b/engine/schema/src/main/java/com/cloud/dc/ClusterDetailsVO.java index 6eb9e7466a7..b213f8f2594 100644 --- a/engine/schema/src/main/java/com/cloud/dc/ClusterDetailsVO.java +++ b/engine/schema/src/main/java/com/cloud/dc/ClusterDetailsVO.java @@ -23,11 +23,11 @@ import javax.persistence.GenerationType; import javax.persistence.Id; import javax.persistence.Table; -import org.apache.cloudstack.api.InternalIdentity; +import org.apache.cloudstack.api.ResourceDetail; @Entity @Table(name = "cluster_details") -public class ClusterDetailsVO implements InternalIdentity { +public class ClusterDetailsVO implements ResourceDetail { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) @@ -35,7 +35,7 @@ public class ClusterDetailsVO implements InternalIdentity { private long id; @Column(name = "cluster_id") - private long clusterId; + private long resourceId; @Column(name = "name") private String name; @@ -47,13 +47,14 @@ public class ClusterDetailsVO implements InternalIdentity { } public ClusterDetailsVO(long clusterId, String name, String value) { - this.clusterId = clusterId; + this.resourceId = clusterId; this.name = name; this.value = value; } - public long getClusterId() { - return clusterId; + @Override + public long getResourceId() { + return resourceId; } public String getName() { @@ -64,6 +65,11 @@ public class ClusterDetailsVO implements InternalIdentity { return value; } + @Override + public boolean isDisplay() { + return true; + } + public void setValue(String value) { this.value = value; } diff --git a/engine/schema/src/main/java/com/cloud/domain/DomainDetailVO.java b/engine/schema/src/main/java/com/cloud/domain/DomainDetailVO.java index df5a2283baa..6f803cc9f2f 100644 --- a/engine/schema/src/main/java/com/cloud/domain/DomainDetailVO.java +++ b/engine/schema/src/main/java/com/cloud/domain/DomainDetailVO.java @@ -23,18 +23,18 @@ import javax.persistence.GenerationType; import javax.persistence.Id; import javax.persistence.Table; -import org.apache.cloudstack.api.InternalIdentity; +import org.apache.cloudstack.api.ResourceDetail; @Entity @Table(name = "domain_details") -public class DomainDetailVO implements InternalIdentity { +public class DomainDetailVO implements ResourceDetail { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) @Column(name = "id") private long id; @Column(name = "domain_id") - private long domainId; + private long resourceId; @Column(name = "name") private String name; @@ -46,13 +46,14 @@ public class DomainDetailVO implements InternalIdentity { } public DomainDetailVO(long domainId, String name, String value) { - this.domainId = domainId; + this.resourceId = domainId; this.name = name; this.value = value; } - public long getDomainId() { - return domainId; + @Override + public long getResourceId() { + return resourceId; } public String getName() { @@ -63,6 +64,11 @@ public class DomainDetailVO implements InternalIdentity { return value; } + @Override + public boolean isDisplay() { + return true; + } + public void setValue(String value) { this.value = value; } diff --git a/engine/schema/src/main/java/com/cloud/domain/dao/DomainDetailsDao.java b/engine/schema/src/main/java/com/cloud/domain/dao/DomainDetailsDao.java index 6b53e49764e..ae149ff4381 100644 --- a/engine/schema/src/main/java/com/cloud/domain/dao/DomainDetailsDao.java +++ b/engine/schema/src/main/java/com/cloud/domain/dao/DomainDetailsDao.java @@ -20,8 +20,9 @@ import java.util.Map; import com.cloud.domain.DomainDetailVO; import com.cloud.utils.db.GenericDao; +import org.apache.cloudstack.resourcedetail.ResourceDetailsDao; -public interface DomainDetailsDao extends GenericDao { +public interface DomainDetailsDao extends GenericDao, ResourceDetailsDao { Map findDetails(long domainId); void persist(long domainId, Map details); @@ -31,6 +32,4 @@ public interface DomainDetailsDao extends GenericDao { void deleteDetails(long domainId); void update(long domainId, Map details); - - String getActualValue(DomainDetailVO domainDetailVO); } diff --git a/engine/schema/src/main/java/com/cloud/domain/dao/DomainDetailsDaoImpl.java b/engine/schema/src/main/java/com/cloud/domain/dao/DomainDetailsDaoImpl.java index b9721a2e58c..5b4e4c591ff 100644 --- a/engine/schema/src/main/java/com/cloud/domain/dao/DomainDetailsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/domain/dao/DomainDetailsDaoImpl.java @@ -25,19 +25,17 @@ import javax.inject.Inject; import org.apache.cloudstack.framework.config.ConfigKey.Scope; import org.apache.cloudstack.framework.config.ScopedConfigStorage; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; -import org.apache.cloudstack.framework.config.impl.ConfigurationVO; import com.cloud.domain.DomainDetailVO; import com.cloud.domain.DomainVO; -import com.cloud.utils.crypt.DBEncryptionUtil; -import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.QueryBuilder; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.SearchCriteria.Op; import com.cloud.utils.db.TransactionLegacy; +import org.apache.cloudstack.resourcedetail.ResourceDetailsDaoBase; -public class DomainDetailsDaoImpl extends GenericDaoBase implements DomainDetailsDao, ScopedConfigStorage { +public class DomainDetailsDaoImpl extends ResourceDetailsDaoBase implements DomainDetailsDao, ScopedConfigStorage { protected final SearchBuilder domainSearch; @Inject @@ -47,14 +45,14 @@ public class DomainDetailsDaoImpl extends GenericDaoBase i protected DomainDetailsDaoImpl() { domainSearch = createSearchBuilder(); - domainSearch.and("domainId", domainSearch.entity().getDomainId(), Op.EQ); + domainSearch.and("domainId", domainSearch.entity().getResourceId(), Op.EQ); domainSearch.done(); } @Override public Map findDetails(long domainId) { QueryBuilder sc = QueryBuilder.create(DomainDetailVO.class); - sc.and(sc.entity().getDomainId(), Op.EQ, domainId); + sc.and(sc.entity().getResourceId(), Op.EQ, domainId); List results = sc.list(); Map details = new HashMap(results.size()); for (DomainDetailVO r : results) { @@ -80,11 +78,16 @@ public class DomainDetailsDaoImpl extends GenericDaoBase i @Override public DomainDetailVO findDetail(long domainId, String name) { QueryBuilder sc = QueryBuilder.create(DomainDetailVO.class); - sc.and(sc.entity().getDomainId(), Op.EQ, domainId); + sc.and(sc.entity().getResourceId(), Op.EQ, domainId); sc.and(sc.entity().getName(), Op.EQ, name); return sc.find(); } + @Override + public void addDetail(long resourceId, String key, String value, boolean display) { + super.addDetail(new DomainDetailVO(resourceId, key, value)); + } + @Override public void deleteDetails(long domainId) { SearchCriteria sc = domainSearch.create(); @@ -129,13 +132,4 @@ public class DomainDetailsDaoImpl extends GenericDaoBase i } return vo == null ? null : getActualValue(vo); } - - @Override - public String getActualValue(DomainDetailVO domainDetailVO) { - ConfigurationVO configurationVO = _configDao.findByName(domainDetailVO.getName()); - if (configurationVO != null && configurationVO.isEncrypted()) { - return DBEncryptionUtil.decrypt(domainDetailVO.getValue()); - } - return domainDetailVO.getValue(); - } } diff --git a/engine/schema/src/main/java/com/cloud/network/dao/FirewallRulesDao.java b/engine/schema/src/main/java/com/cloud/network/dao/FirewallRulesDao.java index e5c3a661bcd..7f322ae6c03 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/FirewallRulesDao.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/FirewallRulesDao.java @@ -43,7 +43,9 @@ public interface FirewallRulesDao extends GenericDao { List listStaticNatByVmId(long vmId); - List listByIpPurposeAndProtocolAndNotRevoked(long ipAddressId, Integer startPort, Integer endPort, String protocol, FirewallRule.Purpose purpose); + List listByIpPurposePortsProtocolAndNotRevoked(long ipAddressId, Integer startPort, Integer endPort, String protocol, FirewallRule.Purpose purpose); + + List listByIpPurposeProtocolAndNotRevoked(long ipAddressId, FirewallRule.Purpose purpose, String protocol); FirewallRuleVO findByRelatedId(long ruleId); diff --git a/engine/schema/src/main/java/com/cloud/network/dao/FirewallRulesDaoImpl.java b/engine/schema/src/main/java/com/cloud/network/dao/FirewallRulesDaoImpl.java index feed641df77..27bf7ba6aa8 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/FirewallRulesDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/FirewallRulesDaoImpl.java @@ -270,8 +270,25 @@ public class FirewallRulesDaoImpl extends GenericDaoBase i } @Override - public List listByIpPurposeAndProtocolAndNotRevoked(long ipAddressId, Integer startPort, Integer endPort, String protocol, - FirewallRule.Purpose purpose) { + public List listByIpPurposeProtocolAndNotRevoked(long ipAddressId, Purpose purpose, String protocol) { + SearchCriteria sc = NotRevokedSearch.create(); + sc.setParameters("ipId", ipAddressId); + sc.setParameters("state", State.Revoke); + + if (purpose != null) { + sc.setParameters("purpose", purpose); + } + + if (protocol != null) { + sc.setParameters("protocol", protocol); + } + + return listBy(sc); + } + + @Override + public List listByIpPurposePortsProtocolAndNotRevoked(long ipAddressId, Integer startPort, Integer endPort, String protocol, + FirewallRule.Purpose purpose) { SearchCriteria sc = NotRevokedSearch.create(); sc.setParameters("ipId", ipAddressId); sc.setParameters("state", State.Revoke); diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/BucketDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/BucketDao.java index f45f28b5c2c..2511df49807 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/BucketDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/BucketDao.java @@ -27,4 +27,8 @@ public interface BucketDao extends GenericDao { List listByObjectStoreIdAndAccountId(long objectStoreId, long accountId); List searchByIds(Long[] ids); + + Long countBucketsForAccount(long accountId); + + Long calculateObjectStorageAllocationForAccount(long accountId); } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/BucketDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/BucketDaoImpl.java index 98bef6201a1..473879d933d 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/BucketDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/BucketDaoImpl.java @@ -16,8 +16,10 @@ // under the License. package com.cloud.storage.dao; +import com.cloud.configuration.Resource; import com.cloud.storage.BucketVO; import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.GenericSearchBuilder; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import org.springframework.stereotype.Component; @@ -31,6 +33,8 @@ public class BucketDaoImpl extends GenericDaoBase implements Buc private SearchBuilder searchFilteringStoreId; private SearchBuilder bucketSearch; + private GenericSearchBuilder CountBucketsByAccount; + private GenericSearchBuilder CalculateBucketsQuotaByAccount; private static final String STORE_ID = "store_id"; private static final String STATE = "state"; @@ -54,6 +58,20 @@ public class BucketDaoImpl extends GenericDaoBase implements Buc bucketSearch.and("idIN", bucketSearch.entity().getId(), SearchCriteria.Op.IN); bucketSearch.done(); + CountBucketsByAccount = createSearchBuilder(Long.class); + CountBucketsByAccount.select(null, SearchCriteria.Func.COUNT, null); + CountBucketsByAccount.and(ACCOUNT_ID, CountBucketsByAccount.entity().getAccountId(), SearchCriteria.Op.EQ); + CountBucketsByAccount.and(STATE, CountBucketsByAccount.entity().getState(), SearchCriteria.Op.NIN); + CountBucketsByAccount.and("removed", CountBucketsByAccount.entity().getRemoved(), SearchCriteria.Op.NULL); + CountBucketsByAccount.done(); + + CalculateBucketsQuotaByAccount = createSearchBuilder(SumCount.class); + CalculateBucketsQuotaByAccount.select("sum", SearchCriteria.Func.SUM, CalculateBucketsQuotaByAccount.entity().getQuota()); + CalculateBucketsQuotaByAccount.and(ACCOUNT_ID, CalculateBucketsQuotaByAccount.entity().getAccountId(), SearchCriteria.Op.EQ); + CalculateBucketsQuotaByAccount.and(STATE, CalculateBucketsQuotaByAccount.entity().getState(), SearchCriteria.Op.NIN); + CalculateBucketsQuotaByAccount.and("removed", CalculateBucketsQuotaByAccount.entity().getRemoved(), SearchCriteria.Op.NULL); + CalculateBucketsQuotaByAccount.done(); + return true; } @Override @@ -79,4 +97,21 @@ public class BucketDaoImpl extends GenericDaoBase implements Buc sc.setParameters("idIN", ids); return search(sc, null, null, false); } + + @Override + public Long countBucketsForAccount(long accountId) { + SearchCriteria sc = CountBucketsByAccount.create(); + sc.setParameters(ACCOUNT_ID, accountId); + sc.setParameters(STATE, BucketVO.State.Destroyed); + return customSearch(sc, null).get(0); + } + + @Override + public Long calculateObjectStorageAllocationForAccount(long accountId) { + SearchCriteria sc = CalculateBucketsQuotaByAccount.create(); + sc.setParameters(ACCOUNT_ID, accountId); + sc.setParameters(STATE, BucketVO.State.Destroyed); + Long totalQuota = customSearch(sc, null).get(0).sum; + return (totalQuota * Resource.ResourceType.bytesToGiB); + } } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolDetailsDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolDetailsDaoImpl.java index 376933f92e7..a3baa3b4cb0 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolDetailsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolDetailsDaoImpl.java @@ -30,6 +30,8 @@ import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailVO; import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import com.cloud.utils.Pair; + public class StoragePoolDetailsDaoImpl extends ResourceDetailsDaoBase implements StoragePoolDetailsDao, ScopedConfigStorage { @Inject @@ -46,7 +48,7 @@ public class StoragePoolDetailsDaoImpl extends ResourceDetailsDaoBase getParentScope(long id) { + StoragePoolVO pool = _storagePoolDao.findById(id); + if (pool != null) { + if (pool.getClusterId() != null) { + return new Pair<>(getScope().getParent(), pool.getClusterId()); + } else { + return new Pair<>(ConfigKey.Scope.Zone, pool.getDataCenterId()); + } + } + return null; + } } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/ConfigurationGroupsAggregator.java b/engine/schema/src/main/java/com/cloud/upgrade/ConfigurationGroupsAggregator.java index 03857137ded..5c1a7504692 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/ConfigurationGroupsAggregator.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/ConfigurationGroupsAggregator.java @@ -54,7 +54,7 @@ public class ConfigurationGroupsAggregator { public void updateConfigurationGroups() { LOG.debug("Updating configuration groups"); - List configs = configDao.listAllIncludingRemoved(); + List configs = configDao.searchPartialConfigurations(); if (CollectionUtils.isEmpty(configs)) { return; } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java index 12049b6f240..9b0e10cbbfc 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java @@ -334,7 +334,7 @@ public class SystemVmTemplateRegistration { } }; - public static boolean validateIfSeeded(String url, String path, String nfsVersion) { + public boolean validateIfSeeded(TemplateDataStoreVO templDataStoreVO, String url, String path, String nfsVersion) { String filePath = null; try { filePath = Files.createTempDirectory(TEMPORARY_SECONDARY_STORE).toString(); @@ -347,6 +347,9 @@ public class SystemVmTemplateRegistration { String templatePath = filePath + File.separator + partialDirPath; File templateProps = new File(templatePath + "/template.properties"); if (templateProps.exists()) { + Pair templateSizes = readTemplatePropertiesSizes(templatePath + "/template.properties"); + updateSeededTemplateDetails(templDataStoreVO.getTemplateId(), templDataStoreVO.getDataStoreId(), + templateSizes.first(), templateSizes.second()); LOGGER.info("SystemVM template already seeded, skipping registration"); return true; } @@ -542,6 +545,21 @@ public class SystemVmTemplateRegistration { } } + public void updateSeededTemplateDetails(long templateId, long storeId, long size, long physicalSize) { + VMTemplateVO template = vmTemplateDao.findById(templateId); + template.setSize(size); + vmTemplateDao.update(template.getId(), template); + + TemplateDataStoreVO templateDataStoreVO = templateDataStoreDao.findByStoreTemplate(storeId, template.getId()); + templateDataStoreVO.setSize(size); + templateDataStoreVO.setPhysicalSize(physicalSize); + templateDataStoreVO.setLastUpdated(new Date(DateUtil.currentGMTTime().getTime())); + boolean updated = templateDataStoreDao.update(templateDataStoreVO.getId(), templateDataStoreVO); + if (!updated) { + throw new CloudRuntimeException("Failed to update template_store_ref entry for seeded systemVM template"); + } + } + public void updateSystemVMEntries(Long templateId, Hypervisor.HypervisorType hypervisorType) { vmInstanceDao.updateSystemVmTemplateId(templateId, hypervisorType); } @@ -555,7 +573,7 @@ public class SystemVmTemplateRegistration { } } - private static void readTemplateProperties(String path, SystemVMTemplateDetails details) { + private static Pair readTemplatePropertiesSizes(String path) { File tmpFile = new File(path); Long size = null; Long physicalSize = 0L; @@ -574,8 +592,13 @@ public class SystemVmTemplateRegistration { } catch (IOException ex) { LOGGER.warn("Failed to read from template.properties", ex); } - details.setSize(size); - details.setPhysicalSize(physicalSize); + return new Pair<>(size, physicalSize); + } + + public static void readTemplateProperties(String path, SystemVMTemplateDetails details) { + Pair templateSizes = readTemplatePropertiesSizes(path); + details.setSize(templateSizes.first()); + details.setPhysicalSize(templateSizes.second()); } private void updateTemplateTablesOnFailure(long templateId) { @@ -799,7 +822,7 @@ public class SystemVmTemplateRegistration { TemplateDataStoreVO templateDataStoreVO = templateDataStoreDao.findByStoreTemplate(storeUrlAndId.second(), templateId); if (templateDataStoreVO != null) { String installPath = templateDataStoreVO.getInstallPath(); - if (validateIfSeeded(storeUrlAndId.first(), installPath, nfsVersion)) { + if (validateIfSeeded(templateDataStoreVO, storeUrlAndId.first(), installPath, nfsVersion)) { continue; } } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java index 0b973d195de..223d7a46637 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java @@ -87,6 +87,36 @@ public class DatabaseAccessObject { return columnExists; } + public String getColumnType(Connection conn, String tableName, String columnName) { + try (PreparedStatement pstmt = conn.prepareStatement(String.format("DESCRIBE %s %s", tableName, columnName));){ + ResultSet rs = pstmt.executeQuery(); + if (rs.next()) { + return rs.getString("Type"); + } + } catch (SQLException e) { + logger.warn("Type for column {} can not be retrieved in {} ignoring exception: {}", columnName, tableName, e.getMessage()); + } + return null; + } + + public void addColumn(Connection conn, String tableName, String columnName, String columnDefinition) { + try (PreparedStatement pstmt = conn.prepareStatement(String.format("ALTER TABLE %s ADD COLUMN %s %s", tableName, columnName, columnDefinition));){ + pstmt.executeUpdate(); + logger.debug("Column {} is added successfully from the table {}", columnName, tableName); + } catch (SQLException e) { + logger.warn("Unable to add column {} to table {} due to exception", columnName, tableName, e); + } + } + + public void changeColumn(Connection conn, String tableName, String oldColumnName, String newColumnName, String columnDefinition) { + try (PreparedStatement pstmt = conn.prepareStatement(String.format("ALTER TABLE %s CHANGE COLUMN %s %s %s", tableName, oldColumnName, newColumnName, columnDefinition));){ + pstmt.executeUpdate(); + logger.debug("Column {} is changed successfully to {} from the table {}", oldColumnName, newColumnName, tableName); + } catch (SQLException e) { + logger.warn("Unable to add column {} to {} from the table {} due to exception", oldColumnName, newColumnName, tableName, e); + } + } + public String generateIndexName(String tableName, String... columnName) { return String.format("i_%s__%s", tableName, StringUtils.join(columnName, "__")); } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/DbUpgradeUtils.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/DbUpgradeUtils.java index 2f90422adf8..be073fcce77 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/DbUpgradeUtils.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/DbUpgradeUtils.java @@ -58,4 +58,20 @@ public class DbUpgradeUtils { } } + public static String getTableColumnType(Connection conn, String tableName, String columnName) { + return dao.getColumnType(conn, tableName, columnName); + } + + public static void addTableColumnIfNotExist(Connection conn, String tableName, String columnName, String columnDefinition) { + if (!dao.columnExists(conn, tableName, columnName)) { + dao.addColumn(conn, tableName, columnName, columnDefinition); + } + } + + public static void changeTableColumnIfNotExist(Connection conn, String tableName, String oldColumnName, String newColumnName, String columnDefinition) { + if (dao.columnExists(conn, tableName, oldColumnName)) { + dao.changeColumn(conn, tableName, oldColumnName, newColumnName, columnDefinition); + } + } + } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42010to42100.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42010to42100.java index eb557da58d8..fd52782e57c 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42010to42100.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42010to42100.java @@ -17,6 +17,7 @@ package com.cloud.upgrade.dao; import com.cloud.upgrade.SystemVmTemplateRegistration; +import com.cloud.utils.db.TransactionLegacy; import com.cloud.utils.exception.CloudRuntimeException; import java.io.InputStream; @@ -29,6 +30,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import org.apache.cloudstack.framework.config.ConfigKey; + public class Upgrade42010to42100 extends DbUpgradeAbstractImpl implements DbUpgrade, DbUpgradeSystemVmTemplate { private SystemVmTemplateRegistration systemVmTemplateRegistration; @@ -61,6 +64,7 @@ public class Upgrade42010to42100 extends DbUpgradeAbstractImpl implements DbUpgr @Override public void performDataMigration(Connection conn) { updateKubernetesClusterNodeVersions(conn); + migrateConfigurationScopeToBitmask(conn); } @Override @@ -177,4 +181,35 @@ public class Upgrade42010to42100 extends DbUpgradeAbstractImpl implements DbUpgr } } } + + protected void migrateConfigurationScopeToBitmask(Connection conn) { + String scopeDataType = DbUpgradeUtils.getTableColumnType(conn, "configuration", "scope"); + logger.info("Data type of the column scope of table configuration is {}", scopeDataType); + if (!"varchar(255)".equals(scopeDataType)) { + return; + } + DbUpgradeUtils.addTableColumnIfNotExist(conn, "configuration", "new_scope", "BIGINT DEFAULT 0"); + migrateExistingConfigurationScopeValues(conn); + DbUpgradeUtils.dropTableColumnsIfExist(conn, "configuration", List.of("scope")); + DbUpgradeUtils.changeTableColumnIfNotExist(conn, "configuration", "new_scope", "scope", "BIGINT NOT NULL DEFAULT 0 COMMENT 'Bitmask for scope(s) of this parameter'"); + } + + protected void migrateExistingConfigurationScopeValues(Connection conn) { + StringBuilder sql = new StringBuilder("UPDATE configuration\n" + + "SET new_scope = " + + " CASE "); + for (ConfigKey.Scope scope : ConfigKey.Scope.values()) { + sql.append(" WHEN scope = '").append(scope.name()).append("' THEN ").append(scope.getBitValue()).append(" "); + } + sql.append(" ELSE 0 " + + " END " + + "WHERE scope IS NOT NULL;"); + TransactionLegacy txn = TransactionLegacy.currentTxn(); + try (PreparedStatement pstmt = txn.prepareAutoCloseStatement(sql.toString())) { + pstmt.executeUpdate(); + } catch (SQLException e) { + logger.error("Failed to migrate existing configuration scope values to bitmask", e); + throw new CloudRuntimeException(String.format("Failed to migrate existing configuration scope values to bitmask due to: %s", e.getMessage())); + } + } } diff --git a/engine/schema/src/main/java/com/cloud/user/AccountDetailVO.java b/engine/schema/src/main/java/com/cloud/user/AccountDetailVO.java index 863f6c96008..aa6e49666dd 100644 --- a/engine/schema/src/main/java/com/cloud/user/AccountDetailVO.java +++ b/engine/schema/src/main/java/com/cloud/user/AccountDetailVO.java @@ -23,18 +23,18 @@ import javax.persistence.GenerationType; import javax.persistence.Id; import javax.persistence.Table; -import org.apache.cloudstack.api.InternalIdentity; +import org.apache.cloudstack.api.ResourceDetail; @Entity @Table(name = "account_details") -public class AccountDetailVO implements InternalIdentity { +public class AccountDetailVO implements ResourceDetail { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) @Column(name = "id") private long id; @Column(name = "account_id") - private long accountId; + private long resourceId; @Column(name = "name") private String name; @@ -46,13 +46,14 @@ public class AccountDetailVO implements InternalIdentity { } public AccountDetailVO(long accountId, String name, String value) { - this.accountId = accountId; + this.resourceId = accountId; this.name = name; this.value = value; } - public long getAccountId() { - return accountId; + @Override + public long getResourceId() { + return resourceId; } public String getName() { @@ -63,6 +64,11 @@ public class AccountDetailVO implements InternalIdentity { return value; } + @Override + public boolean isDisplay() { + return true; + } + public void setValue(String value) { this.value = value; } diff --git a/engine/schema/src/main/java/com/cloud/user/AccountDetailsDao.java b/engine/schema/src/main/java/com/cloud/user/AccountDetailsDao.java index 514433e8068..65bbe1670a8 100644 --- a/engine/schema/src/main/java/com/cloud/user/AccountDetailsDao.java +++ b/engine/schema/src/main/java/com/cloud/user/AccountDetailsDao.java @@ -19,8 +19,9 @@ package com.cloud.user; import java.util.Map; import com.cloud.utils.db.GenericDao; +import org.apache.cloudstack.resourcedetail.ResourceDetailsDao; -public interface AccountDetailsDao extends GenericDao { +public interface AccountDetailsDao extends GenericDao, ResourceDetailsDao { Map findDetails(long accountId); void persist(long accountId, Map details); @@ -34,6 +35,4 @@ public interface AccountDetailsDao extends GenericDao { * they will get created */ void update(long accountId, Map details); - - String getActualValue(AccountDetailVO accountDetailVO); } diff --git a/engine/schema/src/main/java/com/cloud/user/AccountDetailsDaoImpl.java b/engine/schema/src/main/java/com/cloud/user/AccountDetailsDaoImpl.java index 510270ad7bf..cbacf9af572 100644 --- a/engine/schema/src/main/java/com/cloud/user/AccountDetailsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/user/AccountDetailsDaoImpl.java @@ -27,22 +27,21 @@ import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.ConfigKey.Scope; import org.apache.cloudstack.framework.config.ScopedConfigStorage; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; -import org.apache.cloudstack.framework.config.impl.ConfigurationVO; import com.cloud.domain.DomainDetailVO; import com.cloud.domain.DomainVO; import com.cloud.domain.dao.DomainDao; import com.cloud.domain.dao.DomainDetailsDao; import com.cloud.user.dao.AccountDao; -import com.cloud.utils.crypt.DBEncryptionUtil; -import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.Pair; import com.cloud.utils.db.QueryBuilder; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.SearchCriteria.Op; import com.cloud.utils.db.TransactionLegacy; +import org.apache.cloudstack.resourcedetail.ResourceDetailsDaoBase; -public class AccountDetailsDaoImpl extends GenericDaoBase implements AccountDetailsDao, ScopedConfigStorage { +public class AccountDetailsDaoImpl extends ResourceDetailsDaoBase implements AccountDetailsDao, ScopedConfigStorage { protected final SearchBuilder accountSearch; @Inject @@ -56,16 +55,16 @@ public class AccountDetailsDaoImpl extends GenericDaoBase protected AccountDetailsDaoImpl() { accountSearch = createSearchBuilder(); - accountSearch.and("accountId", accountSearch.entity().getAccountId(), Op.EQ); + accountSearch.and("accountId", accountSearch.entity().getResourceId(), Op.EQ); accountSearch.done(); } @Override public Map findDetails(long accountId) { QueryBuilder sc = QueryBuilder.create(AccountDetailVO.class); - sc.and(sc.entity().getAccountId(), Op.EQ, accountId); + sc.and(sc.entity().getResourceId(), Op.EQ, accountId); List results = sc.list(); - Map details = new HashMap(results.size()); + Map details = new HashMap<>(results.size()); for (AccountDetailVO r : results) { details.put(r.getName(), r.getValue()); } @@ -89,11 +88,16 @@ public class AccountDetailsDaoImpl extends GenericDaoBase @Override public AccountDetailVO findDetail(long accountId, String name) { QueryBuilder sc = QueryBuilder.create(AccountDetailVO.class); - sc.and(sc.entity().getAccountId(), Op.EQ, accountId); + sc.and(sc.entity().getResourceId(), Op.EQ, accountId); sc.and(sc.entity().getName(), Op.EQ, name); return sc.find(); } + @Override + public void addDetail(long resourceId, String key, String value, boolean display) { + super.addDetail(new AccountDetailVO(resourceId, key, value)); + } + @Override public void deleteDetails(long accountId) { SearchCriteria sc = accountSearch.create(); @@ -155,11 +159,11 @@ public class AccountDetailsDaoImpl extends GenericDaoBase } @Override - public String getActualValue(AccountDetailVO accountDetailVO) { - ConfigurationVO configurationVO = _configDao.findByName(accountDetailVO.getName()); - if (configurationVO != null && configurationVO.isEncrypted()) { - return DBEncryptionUtil.decrypt(accountDetailVO.getValue()); + public Pair getParentScope(long id) { + Account account = _accountDao.findById(id); + if (account == null) { + return null; } - return accountDetailVO.getValue(); + return new Pair<>(getScope().getParent(), account.getDomainId()); } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupScheduleVO.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupScheduleVO.java index fd3c0be18d2..0258c42c52b 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupScheduleVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupScheduleVO.java @@ -58,15 +58,19 @@ public class BackupScheduleVO implements BackupSchedule { @Column(name = "async_job_id") Long asyncJobId; + @Column(name = "max_backups") + Integer maxBackups = 0; + public BackupScheduleVO() { } - public BackupScheduleVO(Long vmId, DateUtil.IntervalType scheduleType, String schedule, String timezone, Date scheduledTimestamp) { + public BackupScheduleVO(Long vmId, DateUtil.IntervalType scheduleType, String schedule, String timezone, Date scheduledTimestamp, Integer maxBackups) { this.vmId = vmId; this.scheduleType = (short) scheduleType.ordinal(); this.schedule = schedule; this.timezone = timezone; this.scheduledTimestamp = scheduledTimestamp; + this.maxBackups = maxBackups; } @Override @@ -128,4 +132,12 @@ public class BackupScheduleVO implements BackupSchedule { public void setAsyncJobId(Long asyncJobId) { this.asyncJobId = asyncJobId; } + + public Integer getMaxBackups() { + return maxBackups; + } + + public void setMaxBackups(Integer maxBackups) { + this.maxBackups = maxBackups; + } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java index b4cd2f7bada..9ef442baff9 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java @@ -88,6 +88,9 @@ public class BackupVO implements Backup { @Column(name = "zone_id") private long zoneId; + @Column(name = "backup_interval_type") + private short backupIntervalType; + @Column(name = "backed_volumes", length = 65535) protected String backedUpVolumes; @@ -208,6 +211,14 @@ public class BackupVO implements Backup { this.zoneId = zoneId; } + public short getBackupIntervalType() { + return backupIntervalType; + } + + public void setBackupIntervalType(short backupIntervalType) { + this.backupIntervalType = backupIntervalType; + } + @Override public Class getEntityType() { return Backup.class; diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDao.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDao.java index 89a13245b0a..ffd5e5a4a66 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDao.java @@ -35,5 +35,10 @@ public interface BackupDao extends GenericDao { List syncBackups(Long zoneId, Long vmId, List externalBackups); BackupVO getBackupVO(Backup backup); List listByOfferingId(Long backupOfferingId); + + List listBackupsByVMandIntervalType(Long vmId, Backup.Type backupType); + BackupResponse newBackupResponse(Backup backup); + public Long countBackupsForAccount(long accountId); + public Long calculateBackupStorageForAccount(long accountId); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDaoImpl.java index 5a9cd062037..b4e1a760282 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDaoImpl.java @@ -24,6 +24,7 @@ import java.util.Objects; import javax.annotation.PostConstruct; import javax.inject.Inject; +import com.cloud.utils.db.GenericSearchBuilder; import org.apache.cloudstack.api.response.BackupResponse; import org.apache.cloudstack.backup.Backup; import org.apache.cloudstack.backup.BackupOffering; @@ -60,6 +61,9 @@ public class BackupDaoImpl extends GenericDaoBase implements Bac BackupOfferingDao backupOfferingDao; private SearchBuilder backupSearch; + private GenericSearchBuilder CountBackupsByAccount; + private GenericSearchBuilder CalculateBackupStorageByAccount; + private SearchBuilder ListBackupsByVMandIntervalType; public BackupDaoImpl() { } @@ -72,6 +76,27 @@ public class BackupDaoImpl extends GenericDaoBase implements Bac backupSearch.and("backup_offering_id", backupSearch.entity().getBackupOfferingId(), SearchCriteria.Op.EQ); backupSearch.and("zone_id", backupSearch.entity().getZoneId(), SearchCriteria.Op.EQ); backupSearch.done(); + + CountBackupsByAccount = createSearchBuilder(Long.class); + CountBackupsByAccount.select(null, SearchCriteria.Func.COUNT, null); + CountBackupsByAccount.and("account", CountBackupsByAccount.entity().getAccountId(), SearchCriteria.Op.EQ); + CountBackupsByAccount.and("status", CountBackupsByAccount.entity().getStatus(), SearchCriteria.Op.NIN); + CountBackupsByAccount.and("removed", CountBackupsByAccount.entity().getRemoved(), SearchCriteria.Op.NULL); + CountBackupsByAccount.done(); + + CalculateBackupStorageByAccount = createSearchBuilder(SumCount.class); + CalculateBackupStorageByAccount.select("sum", SearchCriteria.Func.SUM, CalculateBackupStorageByAccount.entity().getSize()); + CalculateBackupStorageByAccount.and("account", CalculateBackupStorageByAccount.entity().getAccountId(), SearchCriteria.Op.EQ); + CalculateBackupStorageByAccount.and("status", CalculateBackupStorageByAccount.entity().getStatus(), SearchCriteria.Op.NIN); + CalculateBackupStorageByAccount.and("removed", CalculateBackupStorageByAccount.entity().getRemoved(), SearchCriteria.Op.NULL); + CalculateBackupStorageByAccount.done(); + + ListBackupsByVMandIntervalType = createSearchBuilder(); + ListBackupsByVMandIntervalType.and("vmId", ListBackupsByVMandIntervalType.entity().getVmId(), SearchCriteria.Op.EQ); + ListBackupsByVMandIntervalType.and("intervalType", ListBackupsByVMandIntervalType.entity().getBackupIntervalType(), SearchCriteria.Op.EQ); + ListBackupsByVMandIntervalType.and("status", ListBackupsByVMandIntervalType.entity().getStatus(), SearchCriteria.Op.EQ); + ListBackupsByVMandIntervalType.and("removed", ListBackupsByVMandIntervalType.entity().getRemoved(), SearchCriteria.Op.NULL); + ListBackupsByVMandIntervalType.done(); } @Override @@ -142,6 +167,31 @@ public class BackupDaoImpl extends GenericDaoBase implements Bac return listByVmId(zoneId, vmId); } + @Override + public Long countBackupsForAccount(long accountId) { + SearchCriteria sc = CountBackupsByAccount.create(); + sc.setParameters("account", accountId); + sc.setParameters("status", Backup.Status.Error, Backup.Status.Failed, Backup.Status.Removed, Backup.Status.Expunged); + return customSearch(sc, null).get(0); + } + + @Override + public Long calculateBackupStorageForAccount(long accountId) { + SearchCriteria sc = CalculateBackupStorageByAccount.create(); + sc.setParameters("account", accountId); + sc.setParameters("status", Backup.Status.Error, Backup.Status.Failed, Backup.Status.Removed, Backup.Status.Expunged); + return customSearch(sc, null).get(0).sum; + } + + @Override + public List listBackupsByVMandIntervalType(Long vmId, Backup.Type backupType) { + SearchCriteria sc = ListBackupsByVMandIntervalType.create(); + sc.setParameters("vmId", vmId); + sc.setParameters("type", backupType.ordinal()); + sc.setParameters("status", Backup.Status.BackedUp); + return listBy(sc, null); + } + @Override public BackupResponse newBackupResponse(Backup backup) { VMInstanceVO vm = vmInstanceDao.findByIdIncludingRemoved(backup.getVmId()); diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupScheduleDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupScheduleDaoImpl.java index e00ccc5abd7..aac2e3bf232 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupScheduleDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupScheduleDaoImpl.java @@ -97,6 +97,7 @@ public class BackupScheduleDaoImpl extends GenericDaoBase extends GenericDao * Removes all details for the resource specified * @param resourceId */ - public void removeDetails(long resourceId); + void removeDetails(long resourceId); /** @@ -76,7 +76,7 @@ public interface ResourceDetailsDao extends GenericDao * @param resourceId * @return list of details each implementing ResourceDetail interface */ - public List listDetails(long resourceId); + List listDetails(long resourceId); /** * List details for resourceId having display field = forDisplay value passed in @@ -84,21 +84,23 @@ public interface ResourceDetailsDao extends GenericDao * @param forDisplay * @return */ - public List listDetails(long resourceId, boolean forDisplay); + List listDetails(long resourceId, boolean forDisplay); - public Map listDetailsKeyPairs(long resourceId); + Map listDetailsKeyPairs(long resourceId); Map listDetailsKeyPairs(long resourceId, List keys); - public Map listDetailsKeyPairs(long resourceId, boolean forDisplay); + Map listDetailsKeyPairs(long resourceId, boolean forDisplay); Map listDetailsVisibility(long resourceId); - public void saveDetails(List details); + void saveDetails(List details); - public void addDetail(long resourceId, String key, String value, boolean display); + void addDetail(long resourceId, String key, String value, boolean display); - public List findResourceIdsByNameAndValueIn(String name, Object[] values); + List findResourceIdsByNameAndValueIn(String name, Object[] values); - public long batchExpungeForResources(List ids, Long batchSize); + long batchExpungeForResources(List ids, Long batchSize); + + String getActualValue(ResourceDetail resourceDetail); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/ResourceDetailsDaoBase.java b/engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/ResourceDetailsDaoBase.java index f2e156f225a..29d3f88fd90 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/ResourceDetailsDaoBase.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/resourcedetail/ResourceDetailsDaoBase.java @@ -21,9 +21,9 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import org.apache.cloudstack.api.ResourceDetail; import org.apache.commons.collections.CollectionUtils; +import com.cloud.utils.crypt.DBEncryptionUtil; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.GenericSearchBuilder; import com.cloud.utils.db.SearchBuilder; @@ -31,7 +31,17 @@ import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.SearchCriteria.Op; import com.cloud.utils.db.TransactionLegacy; +import org.apache.cloudstack.api.ResourceDetail; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.framework.config.impl.ConfigurationVO; + +import javax.inject.Inject; + public abstract class ResourceDetailsDaoBase extends GenericDaoBase implements ResourceDetailsDao { + + @Inject + private ConfigurationDao configDao; + private SearchBuilder AllFieldsSearch; public ResourceDetailsDaoBase() { @@ -76,8 +86,7 @@ public abstract class ResourceDetailsDaoBase extends G sc.setParameters("value", value); } - List results = search(sc, null); - return results; + return search(sc, null); } public Map listDetailsKeyPairs(long resourceId) { @@ -85,7 +94,7 @@ public abstract class ResourceDetailsDaoBase extends G sc.setParameters("resourceId", resourceId); List results = search(sc, null); - Map details = new HashMap(results.size()); + Map details = new HashMap<>(results.size()); for (R result : results) { details.put(result.getName(), result.getValue()); } @@ -122,8 +131,7 @@ public abstract class ResourceDetailsDaoBase extends G SearchCriteria sc = AllFieldsSearch.create(); sc.setParameters("resourceId", resourceId); - List results = search(sc, null); - return results; + return search(sc, null); } public void removeDetails(long resourceId) { @@ -185,7 +193,7 @@ public abstract class ResourceDetailsDaoBase extends G sc.setParameters("display", forDisplay); List results = search(sc, null); - Map details = new HashMap(results.size()); + Map details = new HashMap<>(results.size()); for (R result : results) { details.put(result.getName(), result.getValue()); } @@ -197,8 +205,7 @@ public abstract class ResourceDetailsDaoBase extends G sc.setParameters("resourceId", resourceId); sc.setParameters("display", forDisplay); - List results = search(sc, null); - return results; + return search(sc, null); } @Override @@ -230,4 +237,13 @@ public abstract class ResourceDetailsDaoBase extends G sc.setParameters("ids", ids.toArray()); return batchExpunge(sc, batchSize); } + + @Override + public String getActualValue(ResourceDetail resourceDetail) { + ConfigurationVO configurationVO = configDao.findByName(resourceDetail.getName()); + if (configurationVO != null && configurationVO.isEncrypted()) { + return DBEncryptionUtil.decrypt(resourceDetail.getValue()); + } + return resourceDetail.getValue(); + } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDetailsDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDetailsDaoImpl.java index 14830490600..ec40dc0dd68 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDetailsDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDetailsDaoImpl.java @@ -20,6 +20,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import javax.inject.Inject; + import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.ConfigKey.Scope; @@ -27,6 +29,8 @@ import org.apache.cloudstack.framework.config.ScopedConfigStorage; import org.apache.cloudstack.resourcedetail.ResourceDetailsDaoBase; import org.springframework.stereotype.Component; +import com.cloud.storage.ImageStore; +import com.cloud.utils.Pair; import com.cloud.utils.crypt.DBEncryptionUtil; import com.cloud.utils.db.QueryBuilder; import com.cloud.utils.db.SearchBuilder; @@ -36,6 +40,8 @@ import com.cloud.utils.db.TransactionLegacy; @Component public class ImageStoreDetailsDaoImpl extends ResourceDetailsDaoBase implements ImageStoreDetailsDao, ScopedConfigStorage { + @Inject + ImageStoreDao imageStoreDao; protected final SearchBuilder storeSearch; @@ -67,7 +73,7 @@ public class ImageStoreDetailsDaoImpl extends ResourceDetailsDaoBase details = listBy(sc); - Map detailsMap = new HashMap(); + Map detailsMap = new HashMap<>(); for (ImageStoreDetailVO detail : details) { String name = detail.getName(); String value = detail.getValue(); @@ -110,9 +116,24 @@ public class ImageStoreDetailsDaoImpl extends ResourceDetailsDaoBase key) { + ImageStoreDetailVO vo = findDetail(id, key.key()); + return vo == null ? null : getActualValue(vo); + } + @Override public void addDetail(long resourceId, String key, String value, boolean display) { super.addDetail(new ImageStoreDetailVO(resourceId, key, value, display)); } + @Override + public Pair getParentScope(long id) { + ImageStore store = imageStoreDao.findById(id); + if (store == null) { + return null; + } + return new Pair<>(getScope().getParent(), store.getDataCenterId()); + } + } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java index 07b0b8b517c..cb7313954dc 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java @@ -755,7 +755,7 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase if (keyword != null) { SearchCriteria ssc = createSearchCriteria(); ssc.addOr("name", SearchCriteria.Op.LIKE, "%" + keyword + "%"); - ssc.addOr("poolType", SearchCriteria.Op.LIKE, new Storage.StoragePoolType("%" + keyword + "%")); + ssc.addOr("poolType", SearchCriteria.Op.LIKE, "%" + keyword + "%"); sc.addAnd("name", SearchCriteria.Op.SC, ssc); } diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42000to42010.sql b/engine/schema/src/main/resources/META-INF/db/schema-42000to42010.sql index 92e0dbb5b2a..bf13e5eee1a 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42000to42010.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42000to42010.sql @@ -24,6 +24,14 @@ CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.user', 'api_key_access', 'boolean DE CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.account', 'api_key_access', 'boolean DEFAULT NULL COMMENT "is api key access allowed for the account" '); CALL `cloud_usage`.`IDEMPOTENT_ADD_COLUMN`('cloud_usage.account', 'api_key_access', 'boolean DEFAULT NULL COMMENT "is api key access allowed for the account" '); +-- Create a new group for Usage Server related configurations +INSERT INTO `cloud`.`configuration_group` (`name`, `description`, `precedence`) VALUES ('Usage Server', 'Usage Server related configuration', 9); +UPDATE `cloud`.`configuration_subgroup` set `group_id` = (SELECT `id` FROM `cloud`.`configuration_group` WHERE `name` = 'Usage Server'), `precedence` = 1 WHERE `name`='Usage'; +UPDATE `cloud`.`configuration` SET `group_id` = (SELECT `id` FROM `cloud`.`configuration_group` WHERE `name` = 'Usage Server') where `subgroup_id` = (SELECT `id` FROM `cloud`.`configuration_subgroup` WHERE `name` = 'Usage'); + +-- Update the description to indicate this setting applies only to volume snapshots on running instances +UPDATE `cloud`.`configuration` SET `description`='whether volume snapshot is enabled on running instances on KVM hosts' WHERE `name`='kvm.snapshot.enabled'; + -- Modify index for mshost_peer DELETE FROM `cloud`.`mshost_peer`; CALL `cloud`.`IDEMPOTENT_DROP_FOREIGN_KEY`('cloud.mshost_peer','fk_mshost_peer__owner_mshost'); diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql b/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql index c357b2815d1..a1d06bd7f2a 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql @@ -19,6 +19,10 @@ -- Schema upgrade from 4.20.1.0 to 4.21.0.0 --; +-- Add columns max_backup and backup_interval_type to backup table +ALTER TABLE `cloud`.`backup_schedule` ADD COLUMN `max_backups` int(8) default NULL COMMENT 'maximum number of backups to maintain'; +ALTER TABLE `cloud`.`backups` ADD COLUMN `backup_interval_type` int(5) COMMENT 'type of backup, e.g. manual, recurring - hourly, daily, weekly or monthly'; + -- Add console_endpoint_creator_address column to cloud.console_session table CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.console_session', 'console_endpoint_creator_address', 'VARCHAR(45)'); diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.account_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.account_view.sql index dc64380fb57..6092fe8e845 100644 --- a/engine/schema/src/main/resources/META-INF/db/views/cloud.account_view.sql +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.account_view.sql @@ -68,6 +68,14 @@ select `primary_storage_count`.`count` AS `primaryStorageTotal`, `secondary_storage_limit`.`max` AS `secondaryStorageLimit`, `secondary_storage_count`.`count` AS `secondaryStorageTotal`, + `backup_limit`.`max` AS `backupLimit`, + `backup_count`.`count` AS `backupTotal`, + `backup_storage_limit`.`max` AS `backupStorageLimit`, + `backup_storage_count`.`count` AS `backupStorageTotal`, + `bucket_limit`.`max` AS `bucketLimit`, + `bucket_count`.`count` AS `bucketTotal`, + `object_storage_limit`.`max` AS `objectStorageLimit`, + `object_storage_count`.`count` AS `objectStorageTotal`, `async_job`.`id` AS `job_id`, `async_job`.`uuid` AS `job_uuid`, `async_job`.`job_status` AS `job_status`, @@ -160,6 +168,30 @@ from `cloud`.`resource_count` secondary_storage_count ON account.id = secondary_storage_count.account_id and secondary_storage_count.type = 'secondary_storage' left join + `cloud`.`resource_limit` backup_limit ON account.id = backup_limit.account_id + and backup_limit.type = 'backup' + left join + `cloud`.`resource_count` backup_count ON account.id = backup_count.account_id + and backup_count.type = 'backup' + left join + `cloud`.`resource_limit` backup_storage_limit ON account.id = backup_storage_limit.account_id + and backup_storage_limit.type = 'backup_storage' + left join + `cloud`.`resource_count` backup_storage_count ON account.id = backup_storage_count.account_id + and backup_storage_count.type = 'backup_storage' + left join + `cloud`.`resource_limit` bucket_limit ON account.id = bucket_limit.account_id + and bucket_limit.type = 'bucket' + left join + `cloud`.`resource_count` bucket_count ON account.id = bucket_count.account_id + and bucket_count.type = 'bucket' + left join + `cloud`.`resource_limit` object_storage_limit ON account.id = object_storage_limit.account_id + and object_storage_limit.type = 'object_storage' + left join + `cloud`.`resource_count` object_storage_count ON account.id = object_storage_count.account_id + and object_storage_count.type = 'object_storage' + left join `cloud`.`async_job` ON async_job.instance_id = account.id and async_job.instance_type = 'Account' and async_job.job_status = 0; diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.domain_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.domain_view.sql index 201ece95023..c9f7bfc51e4 100644 --- a/engine/schema/src/main/resources/META-INF/db/views/cloud.domain_view.sql +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.domain_view.sql @@ -58,7 +58,15 @@ select `primary_storage_limit`.`max` AS `primaryStorageLimit`, `primary_storage_count`.`count` AS `primaryStorageTotal`, `secondary_storage_limit`.`max` AS `secondaryStorageLimit`, - `secondary_storage_count`.`count` AS `secondaryStorageTotal` + `secondary_storage_count`.`count` AS `secondaryStorageTotal`, + `backup_limit`.`max` AS `backupLimit`, + `backup_count`.`count` AS `backupTotal`, + `backup_storage_limit`.`max` AS `backupStorageLimit`, + `backup_storage_count`.`count` AS `backupStorageTotal`, + `bucket_limit`.`max` AS `bucketLimit`, + `bucket_count`.`count` AS `bucketTotal`, + `object_storage_limit`.`max` AS `objectStorageLimit`, + `object_storage_count`.`count` AS `objectStorageTotal` from `cloud`.`domain` left join @@ -132,4 +140,28 @@ from and secondary_storage_limit.type = 'secondary_storage' left join `cloud`.`resource_count` secondary_storage_count ON domain.id = secondary_storage_count.domain_id - and secondary_storage_count.type = 'secondary_storage'; + and secondary_storage_count.type = 'secondary_storage' + left join + `cloud`.`resource_limit` backup_limit ON domain.id = backup_limit.domain_id + and backup_limit.type = 'backup' + left join + `cloud`.`resource_count` backup_count ON domain.id = backup_count.domain_id + and backup_count.type = 'backup' + left join + `cloud`.`resource_limit` backup_storage_limit ON domain.id = backup_storage_limit.domain_id + and backup_storage_limit.type = 'backup_storage' + left join + `cloud`.`resource_count` backup_storage_count ON domain.id = backup_storage_count.domain_id + and backup_storage_count.type = 'backup_storage' + left join + `cloud`.`resource_limit` bucket_limit ON domain.id = bucket_limit.domain_id + and bucket_limit.type = 'bucket' + left join + `cloud`.`resource_count` bucket_count ON domain.id = bucket_count.domain_id + and bucket_count.type = 'bucket' + left join + `cloud`.`resource_limit` object_storage_limit ON domain.id = object_storage_limit.domain_id + and object_storage_limit.type = 'object_storage' + left join + `cloud`.`resource_count` object_storage_count ON domain.id = object_storage_count.domain_id + and object_storage_count.type = 'object_storage'; diff --git a/engine/schema/src/test/java/com/cloud/upgrade/ConfigurationGroupsAggregatorTest.java b/engine/schema/src/test/java/com/cloud/upgrade/ConfigurationGroupsAggregatorTest.java new file mode 100644 index 00000000000..bab36ef00cf --- /dev/null +++ b/engine/schema/src/test/java/com/cloud/upgrade/ConfigurationGroupsAggregatorTest.java @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.upgrade; + +import static org.mockito.Mockito.when; + +import java.util.Collections; + +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.framework.config.dao.ConfigurationGroupDao; +import org.apache.cloudstack.framework.config.dao.ConfigurationSubGroupDao; +import org.apache.cloudstack.framework.config.impl.ConfigurationSubGroupVO; +import org.apache.cloudstack.framework.config.impl.ConfigurationVO; +import org.apache.logging.log4j.Logger; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class ConfigurationGroupsAggregatorTest { + @InjectMocks + private ConfigurationGroupsAggregator configurationGroupsAggregator = new ConfigurationGroupsAggregator(); + + @Mock + private ConfigurationDao configDao; + + @Mock + private ConfigurationGroupDao configGroupDao; + + @Mock + private ConfigurationSubGroupDao configSubGroupDao; + + @Mock + private Logger logger; + + @Test + public void testUpdateConfigurationGroups() { + ConfigurationVO config = new ConfigurationVO("Advanced", "DEFAULT", "management-server", + "test.config.name", null, "description"); + config.setGroupId(1L); + config.setSubGroupId(1L); + + when(configDao.searchPartialConfigurations()).thenReturn(Collections.singletonList(config)); + + ConfigurationSubGroupVO configSubGroup = Mockito.mock(ConfigurationSubGroupVO.class); + when(configSubGroupDao.findByName("name")).thenReturn(configSubGroup); + Mockito.when(configSubGroup.getId()).thenReturn(10L); + Mockito.when(configSubGroup.getGroupId()).thenReturn(5L); + + configurationGroupsAggregator.updateConfigurationGroups(); + + Assert.assertEquals(Long.valueOf(5), config.getGroupId()); + Assert.assertEquals(Long.valueOf(10), config.getSubGroupId()); + Mockito.verify(configDao, Mockito.times(1)).persist(config); + Mockito.verify(logger, Mockito.times(1)).debug("Updating configuration groups"); + Mockito.verify(logger, Mockito.times(1)).debug("Successfully updated configuration groups."); + } +} diff --git a/engine/schema/src/test/java/com/cloud/upgrade/dao/DatabaseAccessObjectTest.java b/engine/schema/src/test/java/com/cloud/upgrade/dao/DatabaseAccessObjectTest.java index 4c07abda938..0c5a99ca05f 100644 --- a/engine/schema/src/test/java/com/cloud/upgrade/dao/DatabaseAccessObjectTest.java +++ b/engine/schema/src/test/java/com/cloud/upgrade/dao/DatabaseAccessObjectTest.java @@ -511,4 +511,57 @@ public class DatabaseAccessObjectTest { verify(loggerMock, times(1)).warn(anyString(), eq(sqlException)); } + @Test + public void testGetColumnType() throws Exception { + when(connectionMock.prepareStatement(contains("DESCRIBE"))).thenReturn(preparedStatementMock); + when(preparedStatementMock.executeQuery()).thenReturn(resultSetMock); + when(resultSetMock.next()).thenReturn(true); + when(resultSetMock.getString("Type")).thenReturn("type"); + + Connection conn = connectionMock; + String tableName = "tableName"; + String columnName = "columnName"; + + Assert.assertEquals("type", dao.getColumnType(conn, tableName, columnName)); + + verify(connectionMock, times(1)).prepareStatement(anyString()); + verify(preparedStatementMock, times(1)).executeQuery(); + verify(preparedStatementMock, times(1)).close(); + verify(loggerMock, times(0)).debug(anyString()); + } + + @Test + public void testAddColumn() throws Exception { + when(connectionMock.prepareStatement(contains("ADD COLUMN"))).thenReturn(preparedStatementMock); + when(preparedStatementMock.executeUpdate()).thenReturn(1); + + Connection conn = connectionMock; + String tableName = "tableName"; + String columnName = "columnName"; + String columnType = "columnType"; + + dao.addColumn(conn, tableName, columnName, columnType); + + verify(connectionMock, times(1)).prepareStatement(anyString()); + verify(preparedStatementMock, times(1)).executeUpdate(); + verify(preparedStatementMock, times(1)).close(); + } + + @Test + public void testChangeColumn() throws Exception { + when(connectionMock.prepareStatement(contains("CHANGE COLUMN"))).thenReturn(preparedStatementMock); + when(preparedStatementMock.executeUpdate()).thenReturn(1); + + Connection conn = connectionMock; + String tableName = "tableName"; + String columnName = "columnName"; + String newColumnName = "columnName2"; + String columnDefinition = "columnDefinition"; + + dao.changeColumn(conn, tableName, columnName, newColumnName, columnDefinition); + + verify(connectionMock, times(1)).prepareStatement(anyString()); + verify(preparedStatementMock, times(1)).executeUpdate(); + verify(preparedStatementMock, times(1)).close(); + } } diff --git a/engine/schema/src/test/java/com/cloud/upgrade/dao/DbUpgradeUtilsTest.java b/engine/schema/src/test/java/com/cloud/upgrade/dao/DbUpgradeUtilsTest.java index 1b775406466..d892b172c10 100644 --- a/engine/schema/src/test/java/com/cloud/upgrade/dao/DbUpgradeUtilsTest.java +++ b/engine/schema/src/test/java/com/cloud/upgrade/dao/DbUpgradeUtilsTest.java @@ -159,4 +159,33 @@ public class DbUpgradeUtilsTest { verify(daoMock, times(1)).columnExists(conn, tableName, column3); verify(daoMock, times(1)).dropColumn(conn, tableName, column3); } + + @Test + public void testAddTableColumnIfNotExist() throws Exception { + Connection conn = connectionMock; + String tableName = "tableName"; + String columnName = "columnName"; + String columnDefinition = "columnDefinition"; + when(daoMock.columnExists(conn, tableName, columnName)).thenReturn(false); + + DbUpgradeUtils.addTableColumnIfNotExist(conn, tableName, columnName, columnDefinition); + + verify(daoMock, times(1)).columnExists(conn, tableName, columnName); + verify(daoMock, times(1)).addColumn(conn, tableName, columnName, columnDefinition); + } + + @Test + public void testChangeTableColumnIfNotExist() throws Exception { + Connection conn = connectionMock; + String tableName = "tableName"; + String oldColumnName = "oldColumnName"; + String newColumnName = "newColumnName"; + String columnDefinition = "columnDefinition"; + when(daoMock.columnExists(conn, tableName, oldColumnName)).thenReturn(true); + + DbUpgradeUtils.changeTableColumnIfNotExist(conn, tableName, oldColumnName, newColumnName, columnDefinition); + + verify(daoMock, times(1)).columnExists(conn, tableName, oldColumnName); + verify(daoMock, times(1)).changeColumn(conn, tableName, oldColumnName, newColumnName, columnDefinition); + } } diff --git a/engine/schema/src/test/java/com/cloud/upgrade/dao/Upgrade42010to42100Test.java b/engine/schema/src/test/java/com/cloud/upgrade/dao/Upgrade42010to42100Test.java new file mode 100644 index 00000000000..035790f0716 --- /dev/null +++ b/engine/schema/src/test/java/com/cloud/upgrade/dao/Upgrade42010to42100Test.java @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.upgrade.dao; + +import static org.mockito.Mockito.when; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.utils.db.TransactionLegacy; + +@RunWith(MockitoJUnitRunner.class) +public class Upgrade42010to42100Test { + @Spy + Upgrade42010to42100 upgrade; + + @Mock + private Connection conn; + + @Test + public void testPerformDataMigration() throws SQLException { + try (MockedStatic ignored = Mockito.mockStatic(DbUpgradeUtils.class)) { + DbUpgradeUtils dbUpgradeUtils = Mockito.mock(DbUpgradeUtils.class); + when(dbUpgradeUtils.getTableColumnType(conn, "configuration", "scope")).thenReturn("varchar(255)"); + + try (MockedStatic ignored2 = Mockito.mockStatic(TransactionLegacy.class)) { + TransactionLegacy txn = Mockito.mock(TransactionLegacy.class); + when(TransactionLegacy.currentTxn()).thenReturn(txn); + PreparedStatement pstmt = Mockito.mock(PreparedStatement.class); + String sql = "UPDATE configuration\n" + + "SET new_scope =" + + " CASE" + + " WHEN scope = 'Global' THEN 1" + + " WHEN scope = 'Zone' THEN 2" + + " WHEN scope = 'Cluster' THEN 4" + + " WHEN scope = 'StoragePool' THEN 8" + + " WHEN scope = 'ManagementServer' THEN 16" + + " WHEN scope = 'ImageStore' THEN 32" + + " WHEN scope = 'Domain' THEN 64" + + " WHEN scope = 'Account' THEN 128" + + " ELSE 0" + + " END WHERE scope IS NOT NULL;"; + when(txn.prepareAutoCloseStatement(sql)).thenReturn(pstmt); + upgrade.performDataMigration(conn); + + Mockito.verify(pstmt, Mockito.times(1)).executeUpdate(); + } + } + } +} diff --git a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java index 3456731ef1c..2f1227a91a5 100644 --- a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java +++ b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java @@ -40,6 +40,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.DataMotionStrategy; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreCapabilities; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreDriver; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; @@ -1534,6 +1535,16 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { verifyFormat(templateInfo.getFormat()); } + // this blurb handles the case where the storage system can clone a volume from a template + String canCloneVolumeFromTemplate = templateInfo.getDataStore().getDriver().getCapabilities().get("CAN_CLONE_VOLUME_FROM_TEMPLATE"); + if (canCloneVolumeFromTemplate != null && canCloneVolumeFromTemplate.toLowerCase().equals("true")) { + DataStoreDriver driver = templateInfo.getDataStore().getDriver(); + driver.createAsync(volumeInfo.getDataStore(), volumeInfo, null); + volumeInfo = _volumeDataFactory.getVolume(volumeInfo.getId(), volumeInfo.getDataStore()); + driver.copyAsync(templateInfo, volumeInfo, null); + return; + } + HostVO hostVO = null; final boolean computeClusterSupportsVolumeClone; @@ -1641,7 +1652,7 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { errMsg = "Create volume from template failed: " + ex.getMessage(); } - throw new CloudRuntimeException(errMsg); + throw new CloudRuntimeException(errMsg, ex); } finally { if (copyCmdAnswer == null) { @@ -2634,7 +2645,7 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { catch (Exception ex) { errMsg = ex.getMessage(); - throw new CloudRuntimeException(errMsg); + throw new CloudRuntimeException(errMsg, ex); } finally { if (copyCmdAnswer == null) { diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImpl.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImpl.java index d2f08260aa3..0fedf746fa6 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImpl.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImpl.java @@ -20,7 +20,6 @@ package org.apache.cloudstack.storage.image.manager; import java.util.ArrayList; import java.util.Collections; -import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -180,28 +179,14 @@ public class ImageStoreProviderManagerImpl implements ImageStoreProviderManager, @Override public DataStore getImageStoreWithFreeCapacity(List imageStores) { - if (imageStores.size() > 1) { - imageStores.sort(new Comparator() { // Sort data stores based on free capacity - @Override - public int compare(DataStore store1, DataStore store2) { - return Long.compare(_statsCollector.imageStoreCurrentFreeCapacity(store1), - _statsCollector.imageStoreCurrentFreeCapacity(store2)); - } - }); - for (DataStore imageStore : imageStores) { - // Return image store if used percentage is less then threshold value i.e. 90%. - if (_statsCollector.imageStoreHasEnoughCapacity(imageStore)) { - return imageStore; - } - } - } else if (imageStores.size() == 1) { - if (_statsCollector.imageStoreHasEnoughCapacity(imageStores.get(0))) { - return imageStores.get(0); + imageStores.sort((store1, store2) -> Long.compare(_statsCollector.imageStoreCurrentFreeCapacity(store2), + _statsCollector.imageStoreCurrentFreeCapacity(store1))); + for (DataStore imageStore : imageStores) { + if (_statsCollector.imageStoreHasEnoughCapacity(imageStore)) { + return imageStore; } } - - // No store with space found - logger.error(String.format("Can't find an image storage in zone with less than %d usage", + logger.error(String.format("Could not find an image storage in zone with less than %d usage", Math.round(_statsCollector.getImageStoreCapacityThreshold() * 100))); return null; } @@ -209,23 +194,11 @@ public class ImageStoreProviderManagerImpl implements ImageStoreProviderManager, @Override public List orderImageStoresOnFreeCapacity(List imageStores) { List stores = new ArrayList<>(); - if (imageStores.size() > 1) { - imageStores.sort(new Comparator() { // Sort data stores based on free capacity - @Override - public int compare(DataStore store1, DataStore store2) { - return Long.compare(_statsCollector.imageStoreCurrentFreeCapacity(store1), - _statsCollector.imageStoreCurrentFreeCapacity(store2)); - } - }); - for (DataStore imageStore : imageStores) { - // Return image store if used percentage is less then threshold value i.e. 90%. - if (_statsCollector.imageStoreHasEnoughCapacity(imageStore)) { - stores.add(imageStore); - } - } - } else if (imageStores.size() == 1) { - if (_statsCollector.imageStoreHasEnoughCapacity(imageStores.get(0))) { - stores.add(imageStores.get(0)); + imageStores.sort((store1, store2) -> Long.compare(_statsCollector.imageStoreCurrentFreeCapacity(store2), + _statsCollector.imageStoreCurrentFreeCapacity(store1))); + for (DataStore imageStore : imageStores) { + if (_statsCollector.imageStoreHasEnoughCapacity(imageStore)) { + stores.add(imageStore); } } return stores; diff --git a/engine/storage/image/src/test/java/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImplTest.java b/engine/storage/image/src/test/java/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImplTest.java index c0462034790..72acd65931a 100644 --- a/engine/storage/image/src/test/java/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImplTest.java +++ b/engine/storage/image/src/test/java/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImplTest.java @@ -16,6 +16,9 @@ // under the License. package org.apache.cloudstack.storage.image.manager; +import com.cloud.server.StatsCollector; +import com.cloud.utils.Pair; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; import org.junit.Assert; @@ -26,14 +29,22 @@ import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + @RunWith(MockitoJUnitRunner.class) public class ImageStoreProviderManagerImplTest { @Mock ImageStoreDao imageStoreDao; + @Mock + StatsCollector statsCollectorMock; + @InjectMocks ImageStoreProviderManagerImpl imageStoreProviderManager = new ImageStoreProviderManagerImpl(); + @Test public void testGetImageStoreZoneId() { final long storeId = 1L; @@ -44,4 +55,56 @@ public class ImageStoreProviderManagerImplTest { long value = imageStoreProviderManager.getImageStoreZoneId(storeId); Assert.assertEquals(zoneId, value); } + + private Pair, List> prepareUnorderedAndOrderedImageStoresForCapacityTests(boolean hasStoragesWithEnoughCapacity) { + DataStore store1 = Mockito.mock(DataStore.class); + Mockito.doReturn(100L).when(statsCollectorMock).imageStoreCurrentFreeCapacity(store1); + Mockito.doReturn(false).when(statsCollectorMock).imageStoreHasEnoughCapacity(store1); + DataStore store2 = Mockito.mock(DataStore.class); + Mockito.doReturn(200L).when(statsCollectorMock).imageStoreCurrentFreeCapacity(store2); + Mockito.doReturn(hasStoragesWithEnoughCapacity).when(statsCollectorMock).imageStoreHasEnoughCapacity(store2); + DataStore store3 = Mockito.mock(DataStore.class); + Mockito.doReturn(300L).when(statsCollectorMock).imageStoreCurrentFreeCapacity(store3); + Mockito.doReturn(hasStoragesWithEnoughCapacity).when(statsCollectorMock).imageStoreHasEnoughCapacity(store3); + DataStore store4 = Mockito.mock(DataStore.class); + Mockito.doReturn(400L).when(statsCollectorMock).imageStoreCurrentFreeCapacity(store4); + Mockito.doReturn(false).when(statsCollectorMock).imageStoreHasEnoughCapacity(store4); + + List unordered = Arrays.asList(store1, store2, store3, store4); + List orderedAndEnoughCapacity = new ArrayList<>(); + if (hasStoragesWithEnoughCapacity) { + orderedAndEnoughCapacity.add(store3); + orderedAndEnoughCapacity.add(store2); + } + + return new Pair<>(unordered, orderedAndEnoughCapacity); + } + + @Test + public void getImageStoreWithFreeCapacityTestImageStoresWithEnoughCapacityExistReturnsImageStoreWithMostFreeCapacity() { + Pair, List> unorderedAndOrdered = prepareUnorderedAndOrderedImageStoresForCapacityTests(true); + + DataStore result = imageStoreProviderManager.getImageStoreWithFreeCapacity(unorderedAndOrdered.first()); + + Assert.assertEquals(unorderedAndOrdered.second().get(0), result); + } + + @Test + public void getImageStoreWithFreeCapacityTestImageStoresWithEnoughCapacityDoNotExistReturnsNull() { + Pair, List> unorderedAndOrdered = prepareUnorderedAndOrderedImageStoresForCapacityTests(false); + + DataStore result = imageStoreProviderManager.getImageStoreWithFreeCapacity(unorderedAndOrdered.first()); + + Assert.assertNull(result); + } + + @Test + public void orderImageStoresOnFreeCapacityTestReturnsImageStoresOrderedFromMostToLeast() { + Pair, List> unorderedAndOrdered = prepareUnorderedAndOrderedImageStoresForCapacityTests(true); + + List result = imageStoreProviderManager.orderImageStoresOnFreeCapacity(unorderedAndOrdered.first()); + + Assert.assertEquals(unorderedAndOrdered.second(), result); + } + } diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java index bf67be91108..26bef607c9b 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java @@ -622,7 +622,7 @@ public class VolumeServiceImpl implements VolumeService { try { Thread.sleep(sleepTime * 1000); } catch (InterruptedException e) { - logger.debug("waiting for template download been interrupted: " + e.toString()); + logger.debug("waiting for template download been interrupted: " + e); } tries--; } @@ -691,7 +691,6 @@ public class VolumeServiceImpl implements VolumeService { } _tmpltPoolDao.releaseFromLockTable(templatePoolRefId); } - return; } protected Void managedCopyBaseImageCallback(AsyncCallbackDispatcher callback, ManagedCreateBaseImageContext context) { @@ -1039,7 +1038,7 @@ public class VolumeServiceImpl implements VolumeService { try { grantAccess(templateOnPrimary, destHost, destPrimaryDataStore); } catch (Exception e) { - throw new StorageAccessException(String.format("Unable to grant access to template: %s on host: %s", templateOnPrimary.getImage(), destHost)); + throw new StorageAccessException(String.format("Unable to grant access to template: %s on host: %s", templateOnPrimary.getImage(), destHost), e); } templateOnPrimary.processEvent(Event.CopyingRequested); @@ -1161,7 +1160,7 @@ public class VolumeServiceImpl implements VolumeService { try { grantAccess(srcTemplateOnPrimary, destHost, destPrimaryDataStore); } catch (Exception e) { - throw new StorageAccessException(String.format("Unable to grant access to src template: %s on host: %s", srcTemplateOnPrimary, destHost)); + throw new StorageAccessException(String.format("Unable to grant access to src template: %s on host: %s", srcTemplateOnPrimary, destHost), e); } _volumeDetailsDao.addDetail(volumeInfo.getId(), volumeDetailKey, String.valueOf(templatePoolRef.getId()), false); @@ -1408,7 +1407,7 @@ public class VolumeServiceImpl implements VolumeService { try { grantAccess(templateOnPrimary, destHost, destPrimaryDataStore); } catch (Exception e) { - throw new StorageAccessException(String.format("Unable to grant access to template: %s on host: %s", templateOnPrimary, destHost)); + throw new StorageAccessException(String.format("Unable to grant access to template: %s on host: %s", templateOnPrimary, destHost), e); } templateOnPrimary.processEvent(Event.CopyingRequested); diff --git a/framework/config/src/main/java/org/apache/cloudstack/config/Configuration.java b/framework/config/src/main/java/org/apache/cloudstack/config/Configuration.java index b93817a9919..d31d14586be 100644 --- a/framework/config/src/main/java/org/apache/cloudstack/config/Configuration.java +++ b/framework/config/src/main/java/org/apache/cloudstack/config/Configuration.java @@ -17,6 +17,9 @@ package org.apache.cloudstack.config; import java.util.Date; +import java.util.List; + +import org.apache.cloudstack.framework.config.ConfigKey; /** * Configuration represents one global configuration parameter for CloudStack. @@ -74,7 +77,9 @@ public interface Configuration { * always global. A non-null value indicates that this parameter can be * set at a certain organization level. */ - String getScope(); + int getScope(); + + List getScopes(); /** * @return can the configuration parameter be changed without restarting the server. diff --git a/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigDepot.java b/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigDepot.java index 5ee5f9dec48..12f3653b9b3 100644 --- a/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigDepot.java +++ b/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigDepot.java @@ -18,6 +18,8 @@ package org.apache.cloudstack.framework.config; import java.util.Set; +import com.cloud.utils.Pair; + /** * ConfigDepot is a repository of configurations. * @@ -34,4 +36,5 @@ public interface ConfigDepot { boolean isNewConfig(ConfigKey configKey); String getConfigStringValue(String key, ConfigKey.Scope scope, Long scopeId); void invalidateConfigCache(String key, ConfigKey.Scope scope, Long scopeId); + Pair getParentScope(ConfigKey.Scope scope, Long id); } diff --git a/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java b/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java index 00cf56345c8..26151ab5b58 100644 --- a/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java +++ b/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java @@ -17,8 +17,14 @@ package org.apache.cloudstack.framework.config; import java.sql.Date; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import org.apache.cloudstack.framework.config.impl.ConfigDepotImpl; +import org.apache.commons.collections.CollectionUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import com.cloud.utils.Pair; import com.cloud.utils.Ternary; @@ -30,6 +36,7 @@ import com.cloud.utils.exception.CloudRuntimeException; * */ public class ConfigKey { + private static final Logger logger = LogManager.getLogger(ConfigKey.class); public static final String CATEGORY_ADVANCED = "Advanced"; public static final String CATEGORY_ALERT = "Alert"; @@ -37,7 +44,89 @@ public class ConfigKey { public static final String CATEGORY_SYSTEM = "System"; public enum Scope { - Global, Zone, Cluster, StoragePool, Account, ManagementServer, ImageStore, Domain + Global(null, 1), + Zone(Global, 1 << 1), + Cluster(Zone, 1 << 2), + StoragePool(Cluster, 1 << 3), + ManagementServer(Global, 1 << 4), + ImageStore(Zone, 1 << 5), + Domain(Global, 1 << 6), + Account(Domain, 1 << 7); + + private final Scope parent; + private final int bitValue; + + Scope(Scope parent, int bitValue) { + this.parent = parent; + this.bitValue = bitValue; + } + + public Scope getParent() { + return parent; + } + + public int getBitValue() { + return bitValue; + } + + public boolean isDescendantOf(Scope other) { + Scope parent = this.getParent(); + while (parent != null) { + if (parent == other) { + return true; + } + parent = parent.getParent(); + } + return false; + } + + public static List getAllDescendants(String str) { + Scope s1 = Scope.valueOf(str); + List scopes = new ArrayList<>(); + for (Scope s : Scope.values()) { + if (s.isDescendantOf(s1)) { + scopes.add(s); + } + } + return scopes; + } + + public static List decode(int bitmask) { + if (bitmask == 0) { + return Collections.emptyList(); + } + List scopes = new ArrayList<>(); + for (Scope scope : Scope.values()) { + if ((bitmask & scope.getBitValue()) != 0) { + scopes.add(scope); + } + } + return scopes; + } + + public static String decodeAsCsv(int bitmask) { + if (bitmask == 0) { + return null; + } + StringBuilder builder = new StringBuilder(); + for (Scope scope : Scope.values()) { + if ((bitmask & scope.getBitValue()) != 0) { + builder.append(scope.name()).append(", "); + } + } + if (builder.length() > 0) { + builder.setLength(builder.length() - 2); + } + return builder.toString(); + } + + public static int getBitmask(Scope... scopes) { + int bitmask = 0; + for (Scope scope : scopes) { + bitmask |= scope.getBitValue(); + } + return bitmask; + } } public enum Kind { @@ -70,8 +159,8 @@ public class ConfigKey { return _displayText; } - public Scope scope() { - return _scope; + public List getScopes() { + return scopes; } public boolean isDynamic() { @@ -108,7 +197,7 @@ public class ConfigKey { private final String _defaultValue; private final String _description; private final String _displayText; - private final Scope _scope; // Parameter can be at different levels (Zone/cluster/pool/account), by default every parameter is at global + private final List scopes; // Parameter can be at different levels (Zone/cluster/pool/account), by default every parameter is at global private final boolean _isDynamic; private final String _parent; private final Ternary _group; // Group name, description with precedence @@ -128,6 +217,10 @@ public class ConfigKey { this(type, name, category, defaultValue, description, isDynamic, scope, null); } + public ConfigKey(String category, Class type, String name, String defaultValue, String description, boolean isDynamic, List scopes) { + this(type, name, category, defaultValue, description, isDynamic, scopes, null); + } + public ConfigKey(String category, Class type, String name, String defaultValue, String description, boolean isDynamic, Scope scope, String parent) { this(type, name, category, defaultValue, description, isDynamic, scope, null, null, parent, null, null, null, null); } @@ -148,6 +241,10 @@ public class ConfigKey { this(type, name, category, defaultValue, description, isDynamic, scope, multiplier, null, null, null, null, null, null); } + public ConfigKey(Class type, String name, String category, String defaultValue, String description, boolean isDynamic, List scopes, T multiplier) { + this(type, name, category, defaultValue, description, isDynamic, scopes, multiplier, null, null, null, null, null, null); + } + public ConfigKey(Class type, String name, String category, String defaultValue, String description, boolean isDynamic, Scope scope, T multiplier, String parent) { this(type, name, category, defaultValue, description, isDynamic, scope, multiplier, null, parent, null, null, null, null); } @@ -159,13 +256,22 @@ public class ConfigKey { public ConfigKey(Class type, String name, String category, String defaultValue, String description, boolean isDynamic, Scope scope, T multiplier, String displayText, String parent, Ternary group, Pair subGroup, Kind kind, String options) { + this(type, name, category, defaultValue, description, isDynamic, scope == null ? null : List.of(scope), multiplier, + displayText, parent, group, subGroup, kind, options); + } + + public ConfigKey(Class type, String name, String category, String defaultValue, String description, boolean isDynamic, List scopes, T multiplier, + String displayText, String parent, Ternary group, Pair subGroup, Kind kind, String options) { _category = category; _type = type; _name = name; _defaultValue = defaultValue; _description = description; _displayText = displayText; - _scope = scope; + this.scopes = new ArrayList<>(); + if (scopes != null) { + this.scopes.addAll(scopes); + } _isDynamic = isDynamic; _multiplier = multiplier; _parent = parent; @@ -218,28 +324,45 @@ public class ConfigKey { String value = s_depot != null ? s_depot.getConfigStringValue(_name, Scope.Global, null) : null; _value = valueOf((value == null) ? defaultValue() : value); } - return _value; } - protected T valueInScope(Scope scope, Long id) { + protected T valueInGlobalOrAvailableParentScope(Scope scope, Long id) { + if (scopes.size() <= 1) { + return value(); + } + Pair s = new Pair<>(scope, id); + do { + s = s_depot != null ? s_depot.getParentScope(s.first(), s.second()) : null; + if (s != null && scopes.contains(s.first())) { + return valueInScope(s.first(), s.second()); + } + } while (s != null); + logger.trace("Global value for config ({}): {}", _name, _value); + return value(); + } + + public T valueInScope(Scope scope, Long id) { if (id == null) { return value(); } - String value = s_depot != null ? s_depot.getConfigStringValue(_name, scope, id) : null; if (value == null) { - return value(); + return valueInGlobalOrAvailableParentScope(scope, id); } + logger.trace("Scope({}) value for config ({}): {}", scope, _name, _value); return valueOf(value); } - public T valueIn(Long id) { - return valueInScope(_scope, id); + protected Scope getPrimaryScope() { + if (CollectionUtils.isNotEmpty(scopes)) { + return scopes.get(0); + } + return null; } - public T valueInDomain(Long domainId) { - return valueInScope(Scope.Domain, domainId); + public T valueIn(Long id) { + return valueInScope(getPrimaryScope(), id); } @SuppressWarnings("unchecked") @@ -277,4 +400,20 @@ public class ConfigKey { } } + public boolean isGlobalOrEmptyScope() { + return CollectionUtils.isEmpty(scopes) || + (scopes.size() == 1 && scopes.get(0) == Scope.Global); + } + + public int getScopeBitmask() { + int bitmask = 0; + if (CollectionUtils.isEmpty(scopes)) { + return bitmask; + } + for (Scope scope : scopes) { + bitmask |= scope.getBitValue(); + } + return bitmask; + } + } diff --git a/framework/config/src/main/java/org/apache/cloudstack/framework/config/ScopedConfigStorage.java b/framework/config/src/main/java/org/apache/cloudstack/framework/config/ScopedConfigStorage.java index 8126b9510a2..7a109456eb0 100644 --- a/framework/config/src/main/java/org/apache/cloudstack/framework/config/ScopedConfigStorage.java +++ b/framework/config/src/main/java/org/apache/cloudstack/framework/config/ScopedConfigStorage.java @@ -18,6 +18,8 @@ package org.apache.cloudstack.framework.config; import org.apache.cloudstack.framework.config.ConfigKey.Scope; +import com.cloud.utils.Pair; + /** * * This method is used by individual storage for configuration @@ -31,4 +33,7 @@ public interface ScopedConfigStorage { default String getConfigValue(long id, ConfigKey key) { return getConfigValue(id, key.key()); } + default Pair getParentScope(long id) { + return null; + } } diff --git a/framework/config/src/main/java/org/apache/cloudstack/framework/config/dao/ConfigurationDao.java b/framework/config/src/main/java/org/apache/cloudstack/framework/config/dao/ConfigurationDao.java index 88569558fc6..c464b12571c 100644 --- a/framework/config/src/main/java/org/apache/cloudstack/framework/config/dao/ConfigurationDao.java +++ b/framework/config/src/main/java/org/apache/cloudstack/framework/config/dao/ConfigurationDao.java @@ -16,6 +16,7 @@ // under the License. package org.apache.cloudstack.framework.config.dao; +import java.util.List; import java.util.Map; import org.apache.cloudstack.framework.config.impl.ConfigurationVO; @@ -67,4 +68,6 @@ public interface ConfigurationDao extends GenericDao { boolean update(String name, String category, String value); void invalidateCache(); + + List searchPartialConfigurations(); } diff --git a/framework/config/src/main/java/org/apache/cloudstack/framework/config/dao/ConfigurationDaoImpl.java b/framework/config/src/main/java/org/apache/cloudstack/framework/config/dao/ConfigurationDaoImpl.java index 7c4a6f9a609..5b941f8fccc 100644 --- a/framework/config/src/main/java/org/apache/cloudstack/framework/config/dao/ConfigurationDaoImpl.java +++ b/framework/config/src/main/java/org/apache/cloudstack/framework/config/dao/ConfigurationDaoImpl.java @@ -43,6 +43,7 @@ public class ConfigurationDaoImpl extends GenericDaoBase InstanceSearch; final SearchBuilder NameSearch; + final SearchBuilder PartialSearch; public static final String UPDATE_CONFIGURATION_SQL = "UPDATE configuration SET value = ? WHERE name = ?"; @@ -53,6 +54,11 @@ public class ConfigurationDaoImpl extends GenericDaoBase searchPartialConfigurations() { + SearchCriteria sc = PartialSearch.create(); + return searchIncludingRemoved(sc, null, null, false); + } } diff --git a/framework/config/src/main/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImpl.java b/framework/config/src/main/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImpl.java index 911a4ad3707..b1c3c5d9a27 100644 --- a/framework/config/src/main/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImpl.java +++ b/framework/config/src/main/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImpl.java @@ -144,9 +144,11 @@ public class ConfigDepotImpl implements ConfigDepot, ConfigDepotAdmin { createOrupdateConfigObject(date, configurable.getConfigComponentName(), key, null); - if ((key.scope() != null) && (key.scope() != ConfigKey.Scope.Global)) { - Set> currentConfigs = _scopeLevelConfigsMap.get(key.scope()); - currentConfigs.add(key); + if (!key.isGlobalOrEmptyScope()) { + for (ConfigKey.Scope scope : key.getScopes()) { + Set> currentConfigs = _scopeLevelConfigsMap.get(scope); + currentConfigs.add(key); + } } } @@ -204,12 +206,12 @@ public class ConfigDepotImpl implements ConfigDepot, ConfigDepotAdmin { } else { boolean configUpdated = false; if (vo.isDynamic() != key.isDynamic() || !ObjectUtils.equals(vo.getDescription(), key.description()) || !ObjectUtils.equals(vo.getDefaultValue(), key.defaultValue()) || - !ObjectUtils.equals(vo.getScope(), key.scope().toString()) || + !ObjectUtils.equals(vo.getScope(), key.getScopeBitmask()) || !ObjectUtils.equals(vo.getComponent(), componentName)) { vo.setDynamic(key.isDynamic()); vo.setDescription(key.description()); vo.setDefaultValue(key.defaultValue()); - vo.setScope(key.scope().toString()); + vo.setScope(key.getScopeBitmask()); vo.setComponent(componentName); vo.setUpdated(date); configUpdated = true; @@ -283,12 +285,7 @@ public class ConfigDepotImpl implements ConfigDepot, ConfigDepotAdmin { scopeId = Long.valueOf(parts[2]); } catch (IllegalArgumentException ignored) {} if (!ConfigKey.Scope.Global.equals(scope) && scopeId != null) { - ScopedConfigStorage scopedConfigStorage = null; - for (ScopedConfigStorage storage : _scopedStorages) { - if (storage.getScope() == scope) { - scopedConfigStorage = storage; - } - } + ScopedConfigStorage scopedConfigStorage = getScopedStorage(scope); if (scopedConfigStorage == null) { throw new CloudRuntimeException("Unable to find config storage for this scope: " + scope + " for " + key); } @@ -315,26 +312,6 @@ public class ConfigDepotImpl implements ConfigDepot, ConfigDepotAdmin { configCache.invalidate(getConfigCacheKey(key, scope, scopeId)); } - public ScopedConfigStorage findScopedConfigStorage(ConfigKey config) { - for (ScopedConfigStorage storage : _scopedStorages) { - if (storage.getScope() == config.scope()) { - return storage; - } - } - - throw new CloudRuntimeException("Unable to find config storage for this scope: " + config.scope() + " for " + config.key()); - } - - public ScopedConfigStorage getDomainScope(ConfigKey config) { - for (ScopedConfigStorage storage : _scopedStorages) { - if (storage.getScope() == ConfigKey.Scope.Domain) { - return storage; - } - } - - throw new CloudRuntimeException("Unable to find config storage for this scope: " + ConfigKey.Scope.Domain + " for " + config.key()); - } - public List getScopedStorages() { return _scopedStorages; } @@ -398,4 +375,27 @@ public class ConfigDepotImpl implements ConfigDepot, ConfigDepotAdmin { public boolean isNewConfig(ConfigKey configKey) { return newConfigs.contains(configKey.key()); } + + protected ScopedConfigStorage getScopedStorage(ConfigKey.Scope scope) { + ScopedConfigStorage scopedConfigStorage = null; + for (ScopedConfigStorage storage : _scopedStorages) { + if (storage.getScope() == scope) { + scopedConfigStorage = storage; + break; + } + } + return scopedConfigStorage; + } + + @Override + public Pair getParentScope(ConfigKey.Scope scope, Long id) { + if (scope.getParent() == null) { + return null; + } + ScopedConfigStorage scopedConfigStorage = getScopedStorage(scope); + if (scopedConfigStorage == null) { + return null; + } + return scopedConfigStorage.getParentScope(id); + } } diff --git a/framework/config/src/main/java/org/apache/cloudstack/framework/config/impl/ConfigurationVO.java b/framework/config/src/main/java/org/apache/cloudstack/framework/config/impl/ConfigurationVO.java index c705cc64072..d12a41864b0 100644 --- a/framework/config/src/main/java/org/apache/cloudstack/framework/config/impl/ConfigurationVO.java +++ b/framework/config/src/main/java/org/apache/cloudstack/framework/config/impl/ConfigurationVO.java @@ -17,6 +17,7 @@ package org.apache.cloudstack.framework.config.impl; import java.util.Date; +import java.util.List; import javax.persistence.Column; import javax.persistence.Entity; @@ -60,7 +61,7 @@ public class ConfigurationVO implements Configuration { private boolean dynamic; @Column(name = "scope") - private String scope; + private Integer scope; @Column(name = "updated") @Temporal(value = TemporalType.TIMESTAMP) @@ -102,6 +103,7 @@ public class ConfigurationVO implements Configuration { this.name = name; this.description = description; this.parent = parentConfigName; + this.scope = 0; setValue(value); setDisplayText(displayText); setGroupId(groupId); @@ -112,7 +114,7 @@ public class ConfigurationVO implements Configuration { this(key.category(), "DEFAULT", component, key.key(), key.defaultValue(), key.description(), key.displayText(), key.parent()); defaultValue = key.defaultValue(); dynamic = key.isDynamic(); - scope = key.scope() != null ? key.scope().toString() : null; + scope = key.getScopeBitmask(); } @Override @@ -183,10 +185,15 @@ public class ConfigurationVO implements Configuration { } @Override - public String getScope() { + public int getScope() { return scope; } + @Override + public List getScopes() { + return ConfigKey.Scope.decode(scope); + } + @Override public boolean isDynamic() { return dynamic; @@ -205,7 +212,7 @@ public class ConfigurationVO implements Configuration { this.defaultValue = defaultValue; } - public void setScope(String scope) { + public void setScope(int scope) { this.scope = scope; } diff --git a/framework/config/src/test/java/org/apache/cloudstack/framework/config/ConfigKeyTest.java b/framework/config/src/test/java/org/apache/cloudstack/framework/config/ConfigKeyTest.java index a3a8aadfa60..50be7200d56 100644 --- a/framework/config/src/test/java/org/apache/cloudstack/framework/config/ConfigKeyTest.java +++ b/framework/config/src/test/java/org/apache/cloudstack/framework/config/ConfigKeyTest.java @@ -16,6 +16,8 @@ // under the License. package org.apache.cloudstack.framework.config; +import java.util.List; + import org.junit.Assert; import org.junit.Test; @@ -47,4 +49,31 @@ public class ConfigKeyTest { ConfigKey key = new ConfigKey("hond", Boolean.class, "naam", "truus", "thrown name", false); Assert.assertFalse("zero and 0L should be considered the same address", key.isSameKeyAs(0L)); } + + @Test + public void testDecode() { + ConfigKey key = new ConfigKey("testcategoey", Boolean.class, "test", "true", "test descriptuin", false, List.of(Scope.Zone, Scope.StoragePool)); + int bitmask = key.getScopeBitmask(); + List scopes = ConfigKey.Scope.decode(bitmask); + Assert.assertEquals(bitmask, ConfigKey.Scope.getBitmask(scopes.toArray(new Scope[0]))); + for (Scope scope : scopes) { + Assert.assertTrue(scope == Scope.Zone || scope == Scope.StoragePool); + } + } + + @Test + public void testDecodeAsCsv() { + ConfigKey key = new ConfigKey("testcategoey", Boolean.class, "test", "true", "test descriptuin", false, List.of(Scope.Zone, Scope.StoragePool)); + int bitmask = key.getScopeBitmask(); + String scopes = ConfigKey.Scope.decodeAsCsv(bitmask); + Assert.assertTrue("Zone, StoragePool".equals(scopes)); + } + + @Test + public void testGetDescendants() { + List descendants = ConfigKey.Scope.getAllDescendants(Scope.Zone.name()); + for (Scope descendant : descendants) { + Assert.assertTrue(descendant == Scope.Cluster || descendant == Scope.StoragePool || descendant == Scope.ImageStore); + } + } } diff --git a/framework/config/src/test/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImplTest.java b/framework/config/src/test/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImplTest.java index 8a7da795345..ed752165aeb 100644 --- a/framework/config/src/test/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImplTest.java +++ b/framework/config/src/test/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImplTest.java @@ -20,10 +20,14 @@ package org.apache.cloudstack.framework.config.impl; import java.util.Collections; import java.util.HashSet; +import java.util.List; import java.util.Set; import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; +import org.apache.cloudstack.framework.config.ScopedConfigStorage; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.framework.config.dao.ConfigurationSubGroupDao; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -33,11 +37,15 @@ import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; import org.springframework.test.util.ReflectionTestUtils; +import com.cloud.utils.Pair; + @RunWith(MockitoJUnitRunner.class) public class ConfigDepotImplTest { @Mock ConfigurationDao _configDao; + @Mock + ConfigurationSubGroupDao configSubGroupDao; @InjectMocks private ConfigDepotImpl configDepotImpl = new ConfigDepotImpl(); @@ -107,4 +115,76 @@ public class ConfigDepotImplTest { runTestGetConfigStringValueExpiry(((ConfigDepotImpl.CONFIG_CACHE_EXPIRE_SECONDS) + 5) * 1000, 2); } + + @Test + public void testPopulateConfigurationNewVO() { + ConfigKey StorageDisableThreshold = new ConfigKey<>(ConfigKey.CATEGORY_ALERT, Double.class, "pool.storage.capacity.disablethreshold", "0.85", + "Percentage (as a value between 0 and 1) of storage utilization above which allocators will disable using the pool for low storage available.", + true, List.of(ConfigKey.Scope.StoragePool, ConfigKey.Scope.Zone)); + Configurable configurable = new Configurable() { + @Override + public String getConfigComponentName() { + return "test"; + } + + @Override + public ConfigKey[] getConfigKeys() { + return new ConfigKey[] { StorageDisableThreshold }; + } + }; + configDepotImpl.setConfigurables(List.of(configurable)); + configDepotImpl.populateConfigurations(); + + Assert.assertEquals("pool.storage.capacity.disablethreshold", + configDepotImpl._scopeLevelConfigsMap.get(ConfigKey.Scope.Zone).iterator().next().key()); + Assert.assertEquals("pool.storage.capacity.disablethreshold", + configDepotImpl._scopeLevelConfigsMap.get(ConfigKey.Scope.StoragePool).iterator().next().key()); + Assert.assertEquals(0, configDepotImpl._scopeLevelConfigsMap.get(ConfigKey.Scope.Cluster).size()); + } + + @Test + public void testPopulateConfiguration() { + ConfigKey StorageDisableThreshold = new ConfigKey<>(ConfigKey.CATEGORY_ALERT, Double.class, "pool.storage.capacity.disablethreshold", "0.85", + "Percentage (as a value between 0 and 1) of storage utilization above which allocators will disable using the pool for low storage available.", + true, List.of(ConfigKey.Scope.StoragePool, ConfigKey.Scope.Zone)); + Configurable configurable = new Configurable() { + @Override + public String getConfigComponentName() { + return "test"; + } + + @Override + public ConfigKey[] getConfigKeys() { + return new ConfigKey[]{StorageDisableThreshold}; + } + }; + configDepotImpl.setConfigurables(List.of(configurable)); + + ConfigurationVO configurationVO = new ConfigurationVO(StorageDisableThreshold.category(), "DEFAULT", "component", + StorageDisableThreshold.key(), StorageDisableThreshold.defaultValue(), StorageDisableThreshold.description(), + StorageDisableThreshold.displayText(), StorageDisableThreshold.parent(), 1L, 10L); + Mockito.when(_configDao.findById("pool.storage.capacity.disablethreshold")).thenReturn(configurationVO); + configDepotImpl.populateConfigurations(); + + Mockito.verify(_configDao, Mockito.times(1)).persist(configurationVO); + } + + @Test + public void getParentScopeWithValidScope() { + ConfigKey.Scope scope = ConfigKey.Scope.Cluster; + ScopedConfigStorage scopedConfigStorage = Mockito.mock(ScopedConfigStorage.class); + Long id = 1L; + ConfigKey.Scope parentScope = ConfigKey.Scope.Zone; + Long parentId = 2L; + + Mockito.when(scopedConfigStorage.getScope()).thenReturn(scope); + Mockito.when(scopedConfigStorage.getParentScope(id)).thenReturn(new Pair<>(parentScope, parentId)); + + configDepotImpl.setScopedStorages(Collections.singletonList(scopedConfigStorage)); + Pair result = configDepotImpl.getParentScope(scope, id); + + Assert.assertNotNull(result); + Assert.assertEquals(parentScope, result.first()); + Assert.assertEquals(parentId, result.second()); + } } diff --git a/framework/db/src/main/java/com/cloud/utils/db/SearchBase.java b/framework/db/src/main/java/com/cloud/utils/db/SearchBase.java index fcc9ded684d..f2d08aa876e 100644 --- a/framework/db/src/main/java/com/cloud/utils/db/SearchBase.java +++ b/framework/db/src/main/java/com/cloud/utils/db/SearchBase.java @@ -484,6 +484,9 @@ public abstract class SearchBase, T, K> { tableAlias = attr.table; } } + if (op == Op.BINARY_OR) { + sql.append("("); + } sql.append(tableAlias).append(".").append(attr.columnName).append(op.toString()); if (op == Op.IN && params.length == 1) { diff --git a/framework/db/src/main/java/com/cloud/utils/db/SearchCriteria.java b/framework/db/src/main/java/com/cloud/utils/db/SearchCriteria.java index 8affbd5300a..caf88fadb9f 100644 --- a/framework/db/src/main/java/com/cloud/utils/db/SearchCriteria.java +++ b/framework/db/src/main/java/com/cloud/utils/db/SearchCriteria.java @@ -38,7 +38,7 @@ public class SearchCriteria { " NOT BETWEEN ? AND ? ", 2), IN(" IN () ", -1), NOTIN(" NOT IN () ", -1), LIKE(" LIKE ? ", 1), NLIKE(" NOT LIKE ? ", 1), NIN(" NOT IN () ", -1), NULL(" IS NULL ", 0), NNULL( " IS NOT NULL ", - 0), SC(" () ", 1), TEXT(" () ", 1), RP("", 0), AND(" AND ", 0), OR(" OR ", 0), NOT(" NOT ", 0), FIND_IN_SET(" ) ", 1); + 0), SC(" () ", 1), TEXT(" () ", 1), RP("", 0), AND(" AND ", 0), OR(" OR ", 0), NOT(" NOT ", 0), FIND_IN_SET(" ) ", 1), BINARY_OR(" & ?) > 0", 1); private final String op; int params; diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/QuotaManagerImpl.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/QuotaManagerImpl.java index c9254814f46..a03f82a4358 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/QuotaManagerImpl.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/QuotaManagerImpl.java @@ -34,6 +34,7 @@ import javax.naming.ConfigurationException; import com.cloud.user.Account; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.quota.activationrule.presetvariables.Configuration; import org.apache.cloudstack.quota.activationrule.presetvariables.GenericPresetVariable; import org.apache.cloudstack.quota.activationrule.presetvariables.PresetVariableHelper; import org.apache.cloudstack.quota.activationrule.presetvariables.PresetVariables; @@ -467,6 +468,11 @@ public class QuotaManagerImpl extends ManagerBase implements QuotaManager { } + Configuration configuration = presetVariables.getConfiguration(); + if (configuration != null) { + jsInterpreter.injectVariable("configuration", configuration.toString()); + } + jsInterpreter.injectStringVariable("resourceType", presetVariables.getResourceType()); jsInterpreter.injectVariable("value", presetVariables.getValue().toString()); jsInterpreter.injectVariable("zone", presetVariables.getZone().toString()); diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/ComputeOffering.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/ComputeOffering.java index 1d294276d47..09182711ca8 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/ComputeOffering.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/ComputeOffering.java @@ -17,10 +17,15 @@ package org.apache.cloudstack.quota.activationrule.presetvariables; +import org.apache.cloudstack.quota.constant.QuotaTypes; + public class ComputeOffering extends GenericPresetVariable { @PresetVariableDefinition(description = "A boolean informing if the compute offering is customized or not.") private boolean customized; + @PresetVariableDefinition(description = "A boolean informing if the compute offering offers HA or not.", supportedTypes = {QuotaTypes.RUNNING_VM}) + private boolean offerHa; + public boolean isCustomized() { return customized; } @@ -30,4 +35,13 @@ public class ComputeOffering extends GenericPresetVariable { fieldNamesToIncludeInToString.add("customized"); } + public boolean offerHa() { + return offerHa; + } + + public void setOfferHa(boolean offerHa) { + this.offerHa = offerHa; + fieldNamesToIncludeInToString.add("offerHa"); + } + } diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Configuration.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Configuration.java new file mode 100644 index 00000000000..e59f78af8d9 --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Configuration.java @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.quota.activationrule.presetvariables; + +import org.apache.cloudstack.quota.constant.QuotaTypes; + +public class Configuration extends GenericPresetVariable{ + + @PresetVariableDefinition(description = "A boolean informing if the cluster configuration force.ha is enabled or not.", supportedTypes = {QuotaTypes.RUNNING_VM}) + private boolean forceHa; + + public boolean getForceHa() { + return forceHa; + } + + public void setForceHa(boolean forceHa) { + this.forceHa = forceHa; + fieldNamesToIncludeInToString.add("forceHa"); + } +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/DiskOfferingPresetVariables.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/DiskOfferingPresetVariables.java new file mode 100644 index 00000000000..b2f5f69502f --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/DiskOfferingPresetVariables.java @@ -0,0 +1,165 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.quota.activationrule.presetvariables; + +public class DiskOfferingPresetVariables extends GenericPresetVariable { + + @PresetVariableDefinition(description = "A long informing the bytes read rate of the disk offering.") + private Long bytesReadRate; + + @PresetVariableDefinition(description = "A long informing the burst bytes read rate of the disk offering.") + private Long bytesReadBurst; + + @PresetVariableDefinition(description = "The length (in seconds) of the bytes read burst.") + private Long bytesReadBurstLength; + + @PresetVariableDefinition(description = "A long informing the bytes write rate of the disk offering.") + private Long bytesWriteRate; + + @PresetVariableDefinition(description = "A long informing the burst bytes write rate of the disk offering.") + private Long bytesWriteBurst; + + @PresetVariableDefinition(description = "The length (in seconds) of the bytes write burst.") + private Long bytesWriteBurstLength; + + @PresetVariableDefinition(description = "A long informing the I/O requests read rate of the disk offering.") + private Long iopsReadRate; + + @PresetVariableDefinition(description = "A long informing the burst I/O requests read rate of the disk offering.") + private Long iopsReadBurst; + + @PresetVariableDefinition(description = "The length (in seconds) of the IOPS read burst.") + private Long iopsReadBurstLength; + + @PresetVariableDefinition(description = "A long informing the I/O requests write rate of the disk offering.") + private Long iopsWriteRate; + + @PresetVariableDefinition(description = "A long informing the burst I/O requests write rate of the disk offering.") + private Long iopsWriteBurst; + + @PresetVariableDefinition(description = "The length (in seconds) of the IOPS write burst.") + private Long iopsWriteBurstLength; + + public Long getBytesReadRate() { + return bytesReadRate; + } + + public void setBytesReadRate(Long bytesReadRate) { + this.bytesReadRate = bytesReadRate; + fieldNamesToIncludeInToString.add("bytesReadRate"); + } + + public Long getBytesReadBurst() { + return bytesReadBurst; + } + + public void setBytesReadBurst(Long bytesReadBurst) { + this.bytesReadBurst = bytesReadBurst; + fieldNamesToIncludeInToString.add("bytesReadBurst"); + } + + public Long getBytesReadBurstLength() { + return bytesReadBurstLength; + } + + public void setBytesReadBurstLength(Long bytesReadBurstLength) { + this.bytesReadBurstLength = bytesReadBurstLength; + fieldNamesToIncludeInToString.add("bytesReadBurstLength"); + } + + public Long getBytesWriteRate() { + return bytesWriteRate; + } + + public void setBytesWriteRate(Long bytesWriteRate) { + this.bytesWriteRate = bytesWriteRate; + fieldNamesToIncludeInToString.add("bytesWriteRate"); + } + + public Long getBytesWriteBurst() { + return bytesWriteBurst; + } + + public void setBytesWriteBurst(Long bytesWriteBurst) { + this.bytesWriteBurst = bytesWriteBurst; + fieldNamesToIncludeInToString.add("bytesWriteBurst"); + } + + public Long getBytesWriteBurstLength() { + return bytesWriteBurstLength; + } + + public void setBytesWriteBurstLength(Long bytesWriteBurstLength) { + this.bytesWriteBurstLength = bytesWriteBurstLength; + fieldNamesToIncludeInToString.add("bytesWriteBurstLength"); + } + + public Long getIopsReadRate() { + return iopsReadRate; + } + + public void setIopsReadRate(Long iopsReadRate) { + this.iopsReadRate = iopsReadRate; + fieldNamesToIncludeInToString.add("iopsReadRate"); + } + + public Long getIopsReadBurst() { + return iopsReadBurst; + } + + public void setIopsReadBurst(Long iopsReadBurst) { + this.iopsReadBurst = iopsReadBurst; + fieldNamesToIncludeInToString.add("iopsReadBurst"); + } + + public Long getIopsReadBurstLength() { + return iopsReadBurstLength; + } + + public void setIopsReadBurstLength(Long iopsReadBurstLength) { + this.iopsReadBurstLength = iopsReadBurstLength; + fieldNamesToIncludeInToString.add("iopsReadBurstLength"); + } + + public Long getIopsWriteRate() { + return iopsWriteRate; + } + + public void setIopsWriteRate(Long iopsWriteRate) { + this.iopsWriteRate = iopsWriteRate; + fieldNamesToIncludeInToString.add("iopsWriteRate"); + } + + public Long getIopsWriteBurst() { + return iopsWriteBurst; + } + + public void setIopsWriteBurst(Long iopsWriteBurst) { + this.iopsWriteBurst = iopsWriteBurst; + fieldNamesToIncludeInToString.add("iopsWriteBurst"); + } + + public Long getIopsWriteBurstLength() { + return iopsWriteBurstLength; + } + + public void setIopsWriteBurstLength(Long iopsWriteBurstLength) { + this.iopsWriteBurstLength = iopsWriteBurstLength; + fieldNamesToIncludeInToString.add("iopsWriteBurstLength"); + } +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java index 1e84ba27e02..05b75f4f64d 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java @@ -25,6 +25,8 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import com.cloud.dc.ClusterDetailsDao; +import com.cloud.dc.ClusterDetailsVO; import com.cloud.host.HostTagVO; import com.cloud.network.dao.NetworkVO; import com.cloud.network.vpc.VpcVO; @@ -37,6 +39,7 @@ import org.apache.cloudstack.acl.dao.RoleDao; import org.apache.cloudstack.backup.BackupOfferingVO; import org.apache.cloudstack.backup.dao.BackupOfferingDao; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.quota.constant.QuotaTypes; import org.apache.cloudstack.quota.dao.NetworkDao; import org.apache.cloudstack.quota.dao.VmTemplateDao; @@ -51,6 +54,7 @@ import org.apache.cloudstack.usage.UsageTypes; import org.apache.cloudstack.utils.bytescale.ByteScaleUtils; import org.apache.cloudstack.utils.jsinterpreter.JsInterpreter; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.ObjectUtils; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; import org.springframework.stereotype.Component; @@ -181,6 +185,11 @@ public class PresetVariableHelper { @Inject VpcDao vpcDao; + @Inject + ConfigurationDao configDao; + + @Inject + ClusterDetailsDao clusterDetailsDao; protected boolean backupSnapshotAfterTakingSnapshot = SnapshotInfo.BackupSnapshotAfterTakingSnapshot.value(); @@ -194,6 +203,7 @@ public class PresetVariableHelper { presetVariables.setAccount(getPresetVariableAccount(usageRecord.getAccountId())); setPresetVariableProject(presetVariables); + setPresetVariableConfiguration(presetVariables, usageRecord); presetVariables.setDomain(getPresetVariableDomain(usageRecord.getDomainId())); presetVariables.setResourceType(usageRecord.getType()); @@ -272,6 +282,39 @@ public class PresetVariableHelper { return zone; } + protected void setPresetVariableConfiguration(PresetVariables presetVariables, UsageVO usageRecord) { + if (usageRecord.getUsageType() != UsageTypes.RUNNING_VM) { + return; + } + + Configuration configuration = new Configuration(); + setForceHaInConfiguration(configuration, usageRecord); + + presetVariables.setConfiguration(configuration); + } + + protected void setForceHaInConfiguration(Configuration configuration, UsageVO usageRecord) { + Long vmId = usageRecord.getUsageId(); + VMInstanceVO vmVo = vmInstanceDao.findByIdIncludingRemoved(vmId); + validateIfObjectIsNull(vmVo, vmId, "VM"); + + Long hostId = ObjectUtils.defaultIfNull(vmVo.getHostId(), vmVo.getLastHostId()); + + HostVO hostVo = hostDao.findByIdIncludingRemoved(hostId); + validateIfObjectIsNull(hostVo, hostId, "host"); + ClusterDetailsVO forceHa = clusterDetailsDao.findDetail(hostVo.getClusterId(), "force.ha"); + + String forceHaValue; + + if (forceHa != null) { + forceHaValue = forceHa.getValue(); + } else { + forceHaValue = configDao.getValue("force.ha"); + } + + configuration.setForceHa((Boolean.parseBoolean(forceHaValue))); + } + protected Value getPresetVariableValue(UsageVO usageRecord) { Long accountId = usageRecord.getAccountId(); int usageType = usageRecord.getUsageType(); @@ -390,12 +433,16 @@ public class PresetVariableHelper { return guestOsVo.getDisplayName(); } - protected ComputeOffering getPresetVariableValueComputeOffering(ServiceOfferingVO serviceOfferingVo) { + protected ComputeOffering getPresetVariableValueComputeOffering(ServiceOfferingVO serviceOfferingVo, int usageType) { ComputeOffering computeOffering = new ComputeOffering(); computeOffering.setId(serviceOfferingVo.getUuid()); computeOffering.setName(serviceOfferingVo.getName()); computeOffering.setCustomized(serviceOfferingVo.isDynamic()); + if (usageType == UsageTypes.RUNNING_VM) { + computeOffering.setOfferHa(serviceOfferingVo.isOfferHA()); + } + return computeOffering; } @@ -404,7 +451,7 @@ public class PresetVariableHelper { long computeOfferingId = vmVo.getServiceOfferingId(); ServiceOfferingVO serviceOfferingVo = serviceOfferingDao.findByIdIncludingRemoved(computeOfferingId); validateIfObjectIsNull(serviceOfferingVo, computeOfferingId, "compute offering"); - value.setComputeOffering(getPresetVariableValueComputeOffering(serviceOfferingVo)); + value.setComputeOffering(getPresetVariableValueComputeOffering(serviceOfferingVo, usageType)); if (usageType == UsageTypes.RUNNING_VM) { value.setComputingResources(getPresetVariableValueComputingResource(vmVo, serviceOfferingVo)); @@ -492,6 +539,7 @@ public class PresetVariableHelper { value.setId(volumeVo.getUuid()); value.setName(volumeVo.getName()); value.setProvisioningType(volumeVo.getProvisioningType()); + value.setVolumeType(volumeVo.getVolumeType()); Long poolId = volumeVo.getPoolId(); if (poolId == null) { @@ -510,13 +558,25 @@ public class PresetVariableHelper { } } - protected GenericPresetVariable getPresetVariableValueDiskOffering(Long diskOfferingId) { + protected DiskOfferingPresetVariables getPresetVariableValueDiskOffering(Long diskOfferingId) { DiskOfferingVO diskOfferingVo = diskOfferingDao.findByIdIncludingRemoved(diskOfferingId); validateIfObjectIsNull(diskOfferingVo, diskOfferingId, "disk offering"); - GenericPresetVariable diskOffering = new GenericPresetVariable(); + DiskOfferingPresetVariables diskOffering = new DiskOfferingPresetVariables(); diskOffering.setId(diskOfferingVo.getUuid()); diskOffering.setName(diskOfferingVo.getName()); + diskOffering.setBytesReadRate(diskOfferingVo.getBytesReadRate()); + diskOffering.setBytesReadBurst(diskOfferingVo.getBytesReadRateMax()); + diskOffering.setBytesReadBurstLength(diskOfferingVo.getBytesReadRateMaxLength()); + diskOffering.setBytesWriteRate(diskOfferingVo.getBytesWriteRate()); + diskOffering.setBytesWriteBurst(diskOfferingVo.getBytesWriteRateMax()); + diskOffering.setBytesWriteBurstLength(diskOfferingVo.getBytesWriteRateMaxLength()); + diskOffering.setIopsReadRate(diskOfferingVo.getIopsReadRate()); + diskOffering.setIopsReadBurst(diskOfferingVo.getIopsReadRateMax()); + diskOffering.setIopsReadBurstLength(diskOfferingVo.getIopsReadRateMaxLength()); + diskOffering.setIopsWriteRate(diskOfferingVo.getIopsWriteRate()); + diskOffering.setIopsWriteBurst(diskOfferingVo.getIopsWriteRateMax()); + diskOffering.setIopsWriteBurstLength(diskOfferingVo.getIopsWriteRateMaxLength()); return diskOffering; } diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariables.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariables.java index 6dab6604e91..1f8b88ca4cd 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariables.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariables.java @@ -39,6 +39,9 @@ public class PresetVariables { @PresetVariableDefinition(description = "Zone where the resource is.") private GenericPresetVariable zone; + @PresetVariableDefinition(description = "Configurations of the resource.") + private Configuration configuration; + @PresetVariableDefinition(description = "A list containing the tariffs ordered by the field 'position'.") private List lastTariffs; @@ -90,6 +93,14 @@ public class PresetVariables { this.zone = zone; } + public Configuration getConfiguration() { + return configuration; + } + + public void setConfiguration(Configuration configuration) { + this.configuration = configuration; + } + public List getLastTariffs() { return lastTariffs; } diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Value.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Value.java index d87146d8798..77e539db0f3 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Value.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Value.java @@ -22,6 +22,7 @@ import java.util.Map; import com.cloud.storage.Snapshot; import com.cloud.storage.Storage.ProvisioningType; +import com.cloud.storage.Volume; import com.cloud.vm.snapshot.VMSnapshot; import org.apache.cloudstack.quota.constant.QuotaTypes; @@ -75,7 +76,7 @@ public class Value extends GenericPresetVariable { private GenericPresetVariable template; @PresetVariableDefinition(description = "Disk offering of the volume.", supportedTypes = {QuotaTypes.VOLUME}) - private GenericPresetVariable diskOffering; + private DiskOfferingPresetVariables diskOffering; @PresetVariableDefinition(description = "Storage where the volume or snapshot is. While handling with snapshots, this value can be from the primary storage if the global " + "setting 'snapshot.backup.to.secondary' is false, otherwise it will be from secondary storage.", supportedTypes = {QuotaTypes.VOLUME, QuotaTypes.SNAPSHOT}) @@ -93,6 +94,10 @@ public class Value extends GenericPresetVariable { @PresetVariableDefinition(description = "The volume format. Values can be: RAW, VHD, VHDX, OVA and QCOW2.", supportedTypes = {QuotaTypes.VOLUME, QuotaTypes.VOLUME_SECONDARY}) private String volumeFormat; + + @PresetVariableDefinition(description = "The volume type. Values can be: UNKNOWN, ROOT, SWAP, DATADISK and ISO.", supportedTypes = {QuotaTypes.VOLUME}) + private Volume.Type volumeType; + private String state; public Host getHost() { @@ -194,11 +199,11 @@ public class Value extends GenericPresetVariable { fieldNamesToIncludeInToString.add("template"); } - public GenericPresetVariable getDiskOffering() { + public DiskOfferingPresetVariables getDiskOffering() { return diskOffering; } - public void setDiskOffering(GenericPresetVariable diskOffering) { + public void setDiskOffering(DiskOfferingPresetVariables diskOffering) { this.diskOffering = diskOffering; fieldNamesToIncludeInToString.add("diskOffering"); } @@ -257,6 +262,15 @@ public class Value extends GenericPresetVariable { return volumeFormat; } + public Volume.Type getVolumeType() { + return volumeType; + } + + public void setVolumeType(Volume.Type volumeType) { + this.volumeType = volumeType; + fieldNamesToIncludeInToString.add("volumeType"); + } + public String getState() { return state; } diff --git a/framework/quota/src/test/java/org/apache/cloudstack/quota/QuotaManagerImplTest.java b/framework/quota/src/test/java/org/apache/cloudstack/quota/QuotaManagerImplTest.java index 5dfc12f7ef8..c62f80d4a44 100644 --- a/framework/quota/src/test/java/org/apache/cloudstack/quota/QuotaManagerImplTest.java +++ b/framework/quota/src/test/java/org/apache/cloudstack/quota/QuotaManagerImplTest.java @@ -270,6 +270,7 @@ public class QuotaManagerImplTest { Mockito.verify(jsInterpreterMock).injectVariable(Mockito.eq("account"), Mockito.anyString()); Mockito.verify(jsInterpreterMock).injectVariable(Mockito.eq("domain"), Mockito.anyString()); Mockito.verify(jsInterpreterMock, Mockito.never()).injectVariable(Mockito.eq("project"), Mockito.anyString()); + Mockito.verify(jsInterpreterMock, Mockito.never()).injectVariable(Mockito.eq("configuration"), Mockito.anyString()); Mockito.verify(jsInterpreterMock).injectStringVariable(Mockito.eq("resourceType"), Mockito.anyString()); Mockito.verify(jsInterpreterMock).injectVariable(Mockito.eq("value"), Mockito.anyString()); Mockito.verify(jsInterpreterMock).injectVariable(Mockito.eq("zone"), Mockito.anyString()); diff --git a/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java b/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java index 7f64939f7bb..e2be3acbbb5 100644 --- a/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java +++ b/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java @@ -27,6 +27,8 @@ import java.util.List; import java.util.Map; import java.util.Set; +import com.cloud.dc.ClusterDetailsDao; +import com.cloud.dc.ClusterDetailsVO; import com.cloud.host.HostTagVO; import com.cloud.hypervisor.Hypervisor; import com.cloud.storage.StoragePoolTagVO; @@ -76,6 +78,7 @@ import com.cloud.storage.SnapshotVO; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.Storage.ProvisioningType; import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.Volume; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.DiskOfferingDao; import com.cloud.storage.dao.GuestOSDao; @@ -122,6 +125,9 @@ public class PresetVariableHelperTest { @Mock HostTagsDao hostTagsDaoMock; + @Mock + ClusterDetailsDao clusterDetailsDaoMock; + @Mock ImageStoreDao imageStoreDaoMock; @@ -208,7 +214,7 @@ public class PresetVariableHelperTest { value.setComputeOffering(getComputeOfferingForTests()); value.setTags(Collections.singletonMap("tag1", "value1")); value.setTemplate(getGenericPresetVariableForTests()); - value.setDiskOffering(getGenericPresetVariableForTests()); + value.setDiskOffering(getDiskOfferingForTests()); value.setProvisioningType(ProvisioningType.THIN); value.setStorage(getStorageForTests()); value.setSize(ByteScaleUtils.GiB); @@ -216,6 +222,7 @@ public class PresetVariableHelperTest { value.setTag("tag_test"); value.setVmSnapshotType(VMSnapshot.Type.Disk); value.setComputingResources(getComputingResourcesForTests()); + value.setVolumeType(Volume.Type.DATADISK); return value; } @@ -232,6 +239,7 @@ public class PresetVariableHelperTest { computeOffering.setId("compute_offering_id"); computeOffering.setName("compute_offering_name"); computeOffering.setCustomized(false); + computeOffering.setOfferHa(false); return computeOffering; } @@ -243,6 +251,14 @@ public class PresetVariableHelperTest { return host; } + private Configuration getConfigurationForTests() { + Configuration configuration = new Configuration(); + configuration.setId("config_id"); + configuration.setName("config_name"); + configuration.setForceHa(false); + return configuration; + } + private List getHostTagsForTests() { return Arrays.asList(new HostTagVO(1, "tag1", false), new HostTagVO(1, "tag2", false)); } @@ -308,6 +324,13 @@ public class PresetVariableHelperTest { return backupOffering; } + private DiskOfferingPresetVariables getDiskOfferingForTests() { + DiskOfferingPresetVariables diskOffering = new DiskOfferingPresetVariables(); + diskOffering.setId("disk_offering_id"); + diskOffering.setName("disk_offering_name"); + return diskOffering; + } + private void mockMethodValidateIfObjectIsNull() { Mockito.doNothing().when(presetVariableHelperSpy).validateIfObjectIsNull(Mockito.any(), Mockito.anyLong(), Mockito.anyString()); } @@ -329,6 +352,7 @@ public class PresetVariableHelperTest { Mockito.doReturn(expected.getAccount()).when(presetVariableHelperSpy).getPresetVariableAccount(Mockito.anyLong()); Mockito.doNothing().when(presetVariableHelperSpy).setPresetVariableProject(Mockito.any()); + Mockito.doNothing().when(presetVariableHelperSpy).setPresetVariableConfiguration(Mockito.any(), Mockito.any()); Mockito.doReturn(expected.getDomain()).when(presetVariableHelperSpy).getPresetVariableDomain(Mockito.anyLong()); Mockito.doReturn(expected.getValue()).when(presetVariableHelperSpy).getPresetVariableValue(Mockito.any(UsageVO.class)); Mockito.doReturn(expected.getZone()).when(presetVariableHelperSpy).getPresetVariableZone(Mockito.anyLong()); @@ -352,6 +376,35 @@ public class PresetVariableHelperTest { Assert.assertNull(result.getProject()); } + @Test + public void setPresetVariableConfigurationTestQuotaTypeDifferentFromRunningVmDoNothing() { + getQuotaTypesForTests(UsageTypes.RUNNING_VM).forEach(type -> { + PresetVariables result = new PresetVariables(); + Mockito.doReturn(type.getKey()).when(usageVoMock).getUsageType(); + presetVariableHelperSpy.setPresetVariableConfiguration(result, usageVoMock); + + Assert.assertNull(result.getConfiguration()); + }); + } + + @Test + public void setPresetVariableConfigurationTestQuotaTypeIsRunningVmSetConfiguration() { + PresetVariables result = new PresetVariables(); + Configuration expectedConfig = getConfigurationForTests(); + HostVO hostVoMock = Mockito.mock(HostVO.class); + ClusterDetailsVO clusterDetailsVoMock = Mockito.mock(ClusterDetailsVO.class); + + Mockito.doReturn(vmInstanceVoMock).when(vmInstanceDaoMock).findByIdIncludingRemoved(Mockito.anyLong()); + Mockito.doReturn(hostVoMock).when(hostDaoMock).findByIdIncludingRemoved(Mockito.anyLong()); + Mockito.doReturn(1L).when(vmInstanceVoMock).getHostId(); + Mockito.doReturn(1).when(usageVoMock).getUsageType(); + Mockito.doReturn(clusterDetailsVoMock).when(clusterDetailsDaoMock).findDetail(Mockito.anyLong(), Mockito.anyString()); + presetVariableHelperSpy.setPresetVariableConfiguration(result, usageVoMock); + + Assert.assertNotNull(result.getConfiguration()); + Assert.assertEquals(expectedConfig.getForceHa(), result.getConfiguration().getForceHa()); + } + @Test public void setPresetVariableProjectTestAccountWithoutRoleSetAsProject() { PresetVariables result = new PresetVariables(); @@ -627,19 +680,36 @@ public class PresetVariableHelperTest { } @Test - public void getPresetVariableValueComputeOfferingTestSetFieldsAndReturnObject() { + public void getPresetVariableValueComputeOfferingForTestSetFieldsAndReturnObjectForRunningVm() { + ComputeOffering expected = getComputeOfferingForTests(); + Mockito.doReturn(expected.getId()).when(serviceOfferingVoMock).getUuid(); + Mockito.doReturn(expected.getName()).when(serviceOfferingVoMock).getName(); + Mockito.doReturn(expected.isCustomized()).when(serviceOfferingVoMock).isDynamic(); + Mockito.doReturn(expected.offerHa()).when(serviceOfferingVoMock).isOfferHA(); + + ComputeOffering result = presetVariableHelperSpy.getPresetVariableValueComputeOffering(serviceOfferingVoMock, UsageTypes.RUNNING_VM); + + assertPresetVariableIdAndName(expected, result); + Assert.assertEquals(expected.isCustomized(), result.isCustomized()); + Assert.assertEquals(expected.offerHa(), result.offerHa()); + validateFieldNamesToIncludeInToString(Arrays.asList("id", "name", "customized", "offerHa"), result); + } + + @Test + public void getPresetVariableValueComputeOfferingForTestSetFieldsAndReturnObjectForAllocatedVm() { ComputeOffering expected = getComputeOfferingForTests(); Mockito.doReturn(expected.getId()).when(serviceOfferingVoMock).getUuid(); Mockito.doReturn(expected.getName()).when(serviceOfferingVoMock).getName(); Mockito.doReturn(expected.isCustomized()).when(serviceOfferingVoMock).isDynamic(); - ComputeOffering result = presetVariableHelperSpy.getPresetVariableValueComputeOffering(serviceOfferingVoMock); + ComputeOffering result = presetVariableHelperSpy.getPresetVariableValueComputeOffering(serviceOfferingVoMock, UsageTypes.ALLOCATED_VM); assertPresetVariableIdAndName(expected, result); Assert.assertEquals(expected.isCustomized(), result.isCustomized()); validateFieldNamesToIncludeInToString(Arrays.asList("id", "name", "customized"), result); } + @Test public void getPresetVariableValueTemplateTestSetValuesAndReturnObject() { VMTemplateVO vmTemplateVoMock = Mockito.mock(VMTemplateVO.class); @@ -698,6 +768,7 @@ public class PresetVariableHelperTest { Mockito.doReturn(expected.getName()).when(volumeVoMock).getName(); Mockito.doReturn(expected.getDiskOffering()).when(presetVariableHelperSpy).getPresetVariableValueDiskOffering(Mockito.anyLong()); Mockito.doReturn(expected.getProvisioningType()).when(volumeVoMock).getProvisioningType(); + Mockito.doReturn(expected.getVolumeType()).when(volumeVoMock).getVolumeType(); Mockito.doReturn(expected.getStorage()).when(presetVariableHelperSpy).getPresetVariableValueStorage(Mockito.anyLong(), Mockito.anyInt()); Mockito.doReturn(expected.getTags()).when(presetVariableHelperSpy).getPresetVariableValueResourceTags(Mockito.anyLong(), Mockito.any(ResourceObjectType.class)); Mockito.doReturn(expected.getSize()).when(volumeVoMock).getSize(); @@ -713,12 +784,13 @@ public class PresetVariableHelperTest { assertPresetVariableIdAndName(expected, result); Assert.assertEquals(expected.getDiskOffering(), result.getDiskOffering()); Assert.assertEquals(expected.getProvisioningType(), result.getProvisioningType()); + Assert.assertEquals(expected.getVolumeType(), result.getVolumeType()); Assert.assertEquals(expected.getStorage(), result.getStorage()); Assert.assertEquals(expected.getTags(), result.getTags()); Assert.assertEquals(expectedSize, result.getSize()); Assert.assertEquals(imageFormat.name(), result.getVolumeFormat()); - validateFieldNamesToIncludeInToString(Arrays.asList("id", "name", "diskOffering", "provisioningType", "storage", "tags", "size", "volumeFormat"), result); + validateFieldNamesToIncludeInToString(Arrays.asList("id", "name", "diskOffering", "provisioningType", "volumeType", "storage", "tags", "size", "volumeFormat"), result); } Mockito.verify(presetVariableHelperSpy, Mockito.times(ImageFormat.values().length)).getPresetVariableValueResourceTags(Mockito.anyLong(), @@ -740,6 +812,7 @@ public class PresetVariableHelperTest { Mockito.doReturn(expected.getName()).when(volumeVoMock).getName(); Mockito.doReturn(expected.getDiskOffering()).when(presetVariableHelperSpy).getPresetVariableValueDiskOffering(Mockito.anyLong()); Mockito.doReturn(expected.getProvisioningType()).when(volumeVoMock).getProvisioningType(); + Mockito.doReturn(expected.getVolumeType()).when(volumeVoMock).getVolumeType(); Mockito.doReturn(expected.getTags()).when(presetVariableHelperSpy).getPresetVariableValueResourceTags(Mockito.anyLong(), Mockito.any(ResourceObjectType.class)); Mockito.doReturn(expected.getSize()).when(volumeVoMock).getSize(); Mockito.doReturn(imageFormat).when(volumeVoMock).getFormat(); @@ -754,12 +827,13 @@ public class PresetVariableHelperTest { assertPresetVariableIdAndName(expected, result); Assert.assertEquals(expected.getDiskOffering(), result.getDiskOffering()); Assert.assertEquals(expected.getProvisioningType(), result.getProvisioningType()); + Assert.assertEquals(expected.getVolumeType(), result.getVolumeType()); Assert.assertNull(result.getStorage()); Assert.assertEquals(expected.getTags(), result.getTags()); Assert.assertEquals(expectedSize, result.getSize()); Assert.assertEquals(imageFormat.name(), result.getVolumeFormat()); - validateFieldNamesToIncludeInToString(Arrays.asList("id", "name", "diskOffering", "provisioningType", "tags", "size", "volumeFormat"), result); + validateFieldNamesToIncludeInToString(Arrays.asList("id", "name", "diskOffering", "provisioningType", "volumeType", "tags", "size", "volumeFormat"), result); } Mockito.verify(presetVariableHelperSpy, Mockito.times(ImageFormat.values().length)).getPresetVariableValueResourceTags(Mockito.anyLong(), @@ -772,14 +846,15 @@ public class PresetVariableHelperTest { Mockito.doReturn(diskOfferingVoMock).when(diskOfferingDaoMock).findByIdIncludingRemoved(Mockito.anyLong()); mockMethodValidateIfObjectIsNull(); - GenericPresetVariable expected = getGenericPresetVariableForTests(); + DiskOfferingPresetVariables expected = getDiskOfferingForTests(); Mockito.doReturn(expected.getId()).when(diskOfferingVoMock).getUuid(); Mockito.doReturn(expected.getName()).when(diskOfferingVoMock).getName(); GenericPresetVariable result = presetVariableHelperSpy.getPresetVariableValueDiskOffering(1l); assertPresetVariableIdAndName(expected, result); - validateFieldNamesToIncludeInToString(Arrays.asList("id", "name"), result); + validateFieldNamesToIncludeInToString(Arrays.asList("bytesReadBurst", "bytesReadBurstLength", "bytesReadRate", "bytesWriteBurst", "bytesWriteBurstLength", "bytesWriteRate", + "id", "iopsReadBurst", "iopsReadBurstLength", "iopsReadRate", "iopsWriteBurst", "iopsWriteBurstLength", "iopsWriteRate", "name"), result); } @Test @@ -1113,7 +1188,7 @@ public class PresetVariableHelperTest { Mockito.doReturn(serviceOfferingVoMock).when(serviceOfferingDaoMock).findByIdIncludingRemoved(Mockito.anyLong()); mockMethodValidateIfObjectIsNull(); - Mockito.doReturn(expected.getComputeOffering()).when(presetVariableHelperSpy).getPresetVariableValueComputeOffering(Mockito.any()); + Mockito.doReturn(expected.getComputeOffering()).when(presetVariableHelperSpy).getPresetVariableValueComputeOffering(Mockito.any(), Mockito.anyInt()); Mockito.doReturn(expected.getComputingResources()).when(presetVariableHelperSpy).getPresetVariableValueComputingResource(Mockito.any(), Mockito.any()); QuotaTypes.listQuotaTypes().forEach((typeInt, value) -> { diff --git a/framework/spring/lifecycle/src/main/java/org/apache/cloudstack/spring/lifecycle/CloudStackExtendedLifeCycle.java b/framework/spring/lifecycle/src/main/java/org/apache/cloudstack/spring/lifecycle/CloudStackExtendedLifeCycle.java index b913033259c..170e3b40e94 100644 --- a/framework/spring/lifecycle/src/main/java/org/apache/cloudstack/spring/lifecycle/CloudStackExtendedLifeCycle.java +++ b/framework/spring/lifecycle/src/main/java/org/apache/cloudstack/spring/lifecycle/CloudStackExtendedLifeCycle.java @@ -39,7 +39,7 @@ import com.cloud.utils.mgmt.ManagementBean; public class CloudStackExtendedLifeCycle extends AbstractBeanCollector { - Map> sorted = new TreeMap>(); + Map> sorted = new TreeMap<>(); public CloudStackExtendedLifeCycle() { super(); @@ -80,13 +80,8 @@ public class CloudStackExtendedLifeCycle extends AbstractBeanCollector { ManagementBean mbean = (ManagementBean)lifecycle; try { JmxUtil.registerMBean(mbean); - } catch (MalformedObjectNameException e) { - logger.warn("Unable to register MBean: " + mbean.getName(), e); - } catch (InstanceAlreadyExistsException e) { - logger.warn("Unable to register MBean: " + mbean.getName(), e); - } catch (MBeanRegistrationException e) { - logger.warn("Unable to register MBean: " + mbean.getName(), e); - } catch (NotCompliantMBeanException e) { + } catch (MalformedObjectNameException | InstanceAlreadyExistsException | + MBeanRegistrationException | NotCompliantMBeanException e) { logger.warn("Unable to register MBean: " + mbean.getName(), e); } logger.info("Registered MBean: " + mbean.getName()); @@ -129,6 +124,7 @@ public class CloudStackExtendedLifeCycle extends AbstractBeanCollector { throw new CloudRuntimeException(e); } catch (Exception e) { logger.error("Error on configuring bean {} - {}", lifecycle.getName(), e.getMessage(), e); + throw new CloudRuntimeException(e); } } }); @@ -141,7 +137,7 @@ public class CloudStackExtendedLifeCycle extends AbstractBeanCollector { Set set = sorted.get(lifecycle.getRunLevel()); if (set == null) { - set = new HashSet(); + set = new HashSet<>(); sorted.put(lifecycle.getRunLevel(), set); } @@ -169,12 +165,7 @@ public class CloudStackExtendedLifeCycle extends AbstractBeanCollector { } } - @Override - public int getPhase() { - return 2000; - } - - private static interface WithComponentLifeCycle { + private interface WithComponentLifeCycle { public void with(ComponentLifecycle lifecycle); } } diff --git a/framework/spring/lifecycle/src/main/java/org/apache/cloudstack/spring/lifecycle/registry/RegistryLifecycle.java b/framework/spring/lifecycle/src/main/java/org/apache/cloudstack/spring/lifecycle/registry/RegistryLifecycle.java index 19d1fe3acc5..00e19304657 100644 --- a/framework/spring/lifecycle/src/main/java/org/apache/cloudstack/spring/lifecycle/registry/RegistryLifecycle.java +++ b/framework/spring/lifecycle/src/main/java/org/apache/cloudstack/spring/lifecycle/registry/RegistryLifecycle.java @@ -48,7 +48,7 @@ public class RegistryLifecycle implements BeanPostProcessor, SmartLifecycle, App * can use this. */ String registryBeanName; - Set beans = new HashSet(); + Set beans = new HashSet<>(); Class typeClass; ApplicationContext applicationContext; Set excludes = null; @@ -79,7 +79,7 @@ public class RegistryLifecycle implements BeanPostProcessor, SmartLifecycle, App protected synchronized void loadExcluded() { Properties props = applicationContext.getBean("DefaultConfigProperties", Properties.class); - excludes = new HashSet(); + excludes = new HashSet<>(); for (String exclude : props.getProperty(EXTENSION_EXCLUDE, "").trim().split("\\s*,\\s*")) { if (StringUtils.hasText(exclude)) { excludes.add(exclude); @@ -109,10 +109,15 @@ public class RegistryLifecycle implements BeanPostProcessor, SmartLifecycle, App while (iter.hasNext()) { Object next = iter.next(); - if (registry.register(next)) { - logger.debug("Registered " + next); - } else { - iter.remove(); + try { + if (registry.register(next)) { + logger.debug("Registered " + next); + } else { + logger.warn("Bean registration failed for " + next.toString()); + iter.remove(); + } + } catch (Throwable e) { + logger.warn("Bean registration attempt resulted in an exception for " + next.toString(), e); } } } diff --git a/plugins/acl/dynamic-role-based/src/main/java/org/apache/cloudstack/acl/DynamicRoleBasedAPIAccessChecker.java b/plugins/acl/dynamic-role-based/src/main/java/org/apache/cloudstack/acl/DynamicRoleBasedAPIAccessChecker.java index 030e0bcf014..6bbd25bb440 100644 --- a/plugins/acl/dynamic-role-based/src/main/java/org/apache/cloudstack/acl/DynamicRoleBasedAPIAccessChecker.java +++ b/plugins/acl/dynamic-role-based/src/main/java/org/apache/cloudstack/acl/DynamicRoleBasedAPIAccessChecker.java @@ -47,7 +47,7 @@ public class DynamicRoleBasedAPIAccessChecker extends AdapterBase implements API private RoleService roleService; private List services; - private Map> annotationRoleBasedApisMap = new HashMap>(); + private Map> annotationRoleBasedApisMap = new HashMap<>(); private LazyCache accountCache; private LazyCache>> rolePermissionsCache; @@ -56,7 +56,7 @@ public class DynamicRoleBasedAPIAccessChecker extends AdapterBase implements API protected DynamicRoleBasedAPIAccessChecker() { super(); for (RoleType roleType : RoleType.values()) { - annotationRoleBasedApisMap.put(roleType, new HashSet()); + annotationRoleBasedApisMap.put(roleType, new HashSet<>()); } } diff --git a/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java b/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java index d4b3cff0f5c..6935d177c72 100644 --- a/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java +++ b/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java @@ -24,6 +24,7 @@ import java.util.Map; import javax.inject.Inject; +import com.cloud.configuration.Resource; import com.cloud.storage.dao.VolumeDao; import org.apache.cloudstack.backup.dao.BackupDao; @@ -99,6 +100,16 @@ public class DummyBackupProvider extends AdapterBase implements BackupProvider { return metrics; } + @Override + public List listRestorePoints(VirtualMachine vm) { + return null; + } + + @Override + public Backup createNewBackupEntryForRestorePoint(Backup.RestorePoint restorePoint, VirtualMachine vm, Backup.Metric metric) { + return null; + } + @Override public boolean removeVMFromBackupOffering(VirtualMachine vm) { logger.debug(String.format("Removing VM %s from backup offering by the Dummy Backup Provider", vm)); @@ -111,7 +122,7 @@ public class DummyBackupProvider extends AdapterBase implements BackupProvider { } @Override - public boolean takeBackup(VirtualMachine vm) { + public Pair takeBackup(VirtualMachine vm) { logger.debug(String.format("Starting backup for VM %s on Dummy provider", vm)); BackupVO backup = new BackupVO(); @@ -119,23 +130,20 @@ public class DummyBackupProvider extends AdapterBase implements BackupProvider { backup.setExternalId("dummy-external-id"); backup.setType("FULL"); backup.setDate(new Date()); - backup.setSize(1024L); - backup.setProtectedSize(1024000L); + backup.setSize(1024000L); + backup.setProtectedSize(1 * Resource.ResourceType.bytesToGiB); backup.setStatus(Backup.Status.BackedUp); backup.setBackupOfferingId(vm.getBackupOfferingId()); backup.setAccountId(vm.getAccountId()); backup.setDomainId(vm.getDomainId()); backup.setZoneId(vm.getDataCenterId()); backup.setBackedUpVolumes(BackupManagerImpl.createVolumeInfoFromVolumes(volumeDao.findByInstance(vm.getId()))); - return backupDao.persist(backup) != null; + backup = backupDao.persist(backup); + return new Pair<>(true, backup); } @Override public boolean deleteBackup(Backup backup, boolean forced) { return true; } - - @Override - public void syncBackups(VirtualMachine vm, Backup.Metric metric) { - } } diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java index 5d3d1a91933..f148c53e614 100644 --- a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java @@ -46,6 +46,7 @@ import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.commons.collections.CollectionUtils; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; + import javax.inject.Inject; import java.text.SimpleDateFormat; import java.util.ArrayList; @@ -141,7 +142,7 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co } @Override - public boolean takeBackup(final VirtualMachine vm) { + public Pair takeBackup(final VirtualMachine vm) { final Host host = getVMHypervisorHost(vm); final BackupRepository backupRepository = backupRepositoryDao.findByBackupOfferingId(vm.getBackupOfferingId()); @@ -179,12 +180,16 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co backupVO.setSize(answer.getSize()); backupVO.setStatus(Backup.Status.BackedUp); backupVO.setBackedUpVolumes(BackupManagerImpl.createVolumeInfoFromVolumes(volumeDao.findByInstance(vm.getId()))); - return backupDao.update(backupVO.getId(), backupVO); + if (backupDao.update(backupVO.getId(), backupVO)) { + return new Pair<>(true, backupVO); + } else { + throw new CloudRuntimeException("Failed to update backup"); + } } else { backupVO.setStatus(Backup.Status.Failed); backupDao.remove(backupVO.getId()); + return new Pair<>(false, null); } - return Objects.nonNull(answer) && answer.getResult(); } private BackupVO createBackupObject(VirtualMachine vm, String backupPath) { @@ -358,6 +363,7 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co return backupDao.remove(backup.getId()); } + logger.debug("There was an error removing the backup with id " + backup.getId()); return false; } @@ -383,6 +389,16 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co return metrics; } + @Override + public List listRestorePoints(VirtualMachine vm) { + return null; + } + + @Override + public Backup createNewBackupEntryForRestorePoint(Backup.RestorePoint restorePoint, VirtualMachine vm, Backup.Metric metric) { + return null; + } + @Override public boolean assignVMToBackupOffering(VirtualMachine vm, BackupOffering backupOffering) { return Hypervisor.HypervisorType.KVM.equals(vm.getHypervisorType()); @@ -398,11 +414,6 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co return false; } - @Override - public void syncBackups(VirtualMachine vm, Backup.Metric metric) { - // TODO: check and sum/return backups metrics on per VM basis - } - @Override public List listBackupOfferings(Long zoneId) { final List repositories = backupRepositoryDao.listByZoneAndProvider(zoneId, getName()); diff --git a/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java b/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java index 393e2911ac3..822688a86a3 100644 --- a/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java +++ b/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java @@ -29,15 +29,11 @@ import com.cloud.storage.dao.VolumeDao; import com.cloud.utils.Pair; import com.cloud.utils.Ternary; import com.cloud.utils.component.AdapterBase; -import com.cloud.utils.db.Transaction; -import com.cloud.utils.db.TransactionCallbackNoReturn; -import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.ssh.SshHelper; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.dao.VMInstanceDao; -import org.apache.cloudstack.api.InternalIdentity; import org.apache.cloudstack.backup.dao.BackupDao; import org.apache.cloudstack.backup.dao.BackupOfferingDaoImpl; import org.apache.cloudstack.backup.networker.NetworkerClient; @@ -462,7 +458,7 @@ public class NetworkerBackupProvider extends AdapterBase implements BackupProvid } @Override - public boolean takeBackup(VirtualMachine vm) { + public Pair takeBackup(VirtualMachine vm) { String networkerServer; String clusterName; @@ -514,11 +510,11 @@ public class NetworkerBackupProvider extends AdapterBase implements BackupProvid if (backup != null) { backup.setBackedUpVolumes(BackupManagerImpl.createVolumeInfoFromVolumes(volumeDao.findByInstance(vm.getId()))); backupDao.persist(backup); - return true; + return new Pair<>(true, backup); } else { LOG.error("Could not register backup for vm {} with saveset Time: {}", vm, saveTime); // We need to handle this rare situation where backup is successful but can't be registered properly. - return false; + return new Pair<>(false, null); } } @@ -532,7 +528,7 @@ public class NetworkerBackupProvider extends AdapterBase implements BackupProvid LOG.debug("EMC Networker successfully deleted backup with id " + externalBackupId); return true; } else { - LOG.debug("There was an error removing the backup with id " + externalBackupId + " from EMC NEtworker"); + LOG.debug("There was an error removing the backup with id " + externalBackupId + " from EMC Networker"); } return false; } @@ -550,12 +546,12 @@ public class NetworkerBackupProvider extends AdapterBase implements BackupProvid for (final VirtualMachine vm : vms) { for ( Backup.VolumeInfo thisVMVol : vm.getBackupVolumeList()) { - vmBackupSize += (thisVMVol.getSize() / 1024L / 1024L); + vmBackupProtectedSize += (thisVMVol.getSize() / 1024L / 1024L); } final ArrayList vmBackups = getClient(zoneId).getBackupsForVm(vm); for ( String vmBackup : vmBackups ) { NetworkerBackup vmNwBackup = getClient(zoneId).getNetworkerBackupInfo(vmBackup); - vmBackupProtectedSize+= vmNwBackup.getSize().getValue() / 1024L; + vmBackupSize += vmNwBackup.getSize().getValue() / 1024L; } Backup.Metric vmBackupMetric = new Backup.Metric(vmBackupSize,vmBackupProtectedSize); LOG.debug(String.format("Metrics for VM [%s] is [backup size: %s, data size: %s].", vm, vmBackupMetric.getBackupSize(), vmBackupMetric.getDataSize())); @@ -565,83 +561,53 @@ public class NetworkerBackupProvider extends AdapterBase implements BackupProvid } @Override - public void syncBackups(VirtualMachine vm, Backup.Metric metric) { - final Long zoneId = vm.getDataCenterId(); - Transaction.execute(new TransactionCallbackNoReturn() { - @Override - public void doInTransactionWithoutResult(TransactionStatus status) { - final List backupsInDb = backupDao.listByVmId(null, vm.getId()); - final ArrayList backupsInNetworker = getClient(zoneId).getBackupsForVm(vm); - final List removeList = backupsInDb.stream().map(InternalIdentity::getId).collect(Collectors.toList()); - for (final String networkerBackupId : backupsInNetworker ) { - Long vmBackupSize=0L; - boolean backupExists = false; - for (final Backup backupInDb : backupsInDb) { - LOG.debug(String.format("Checking if Backup %s with external ID %s for VM %s is valid", backupsInDb, backupInDb.getName(), vm)); - if ( networkerBackupId.equals(backupInDb.getExternalId()) ) { - LOG.debug(String.format("Found Backup %s in both Database and Networker", backupInDb)); - backupExists = true; - removeList.remove(backupInDb.getId()); - if (metric != null) { - LOG.debug(String.format("Update backup [%s] from [size: %s, protected size: %s] to [size: %s, protected size: %s].", - backupInDb, backupInDb.getSize(), backupInDb.getProtectedSize(), - metric.getBackupSize(), metric.getDataSize())); - ((BackupVO) backupInDb).setSize(metric.getBackupSize()); - ((BackupVO) backupInDb).setProtectedSize(metric.getDataSize()); - backupDao.update(backupInDb.getId(), ((BackupVO) backupInDb)); - } - break; - } - } - if (backupExists) { - continue; - } - // Technically an administrator can manually create a backup for a VM by utilizing the KVM scripts - // with the proper parameters. So we will register any backups taken on the Networker side from - // outside Cloudstack. If ever Networker will support KVM out of the box this functionality also will - // ensure that SLA like backups will be found and registered. - NetworkerBackup strayNetworkerBackup = getClient(vm.getDataCenterId()).getNetworkerBackupInfo(networkerBackupId); - // Since running backups are already present in Networker Server but not completed - // make sure the backup is not in progress at this time. - if ( strayNetworkerBackup.getCompletionTime() != null) { - BackupVO strayBackup = new BackupVO(); - strayBackup.setVmId(vm.getId()); - strayBackup.setExternalId(strayNetworkerBackup.getId()); - strayBackup.setType(strayNetworkerBackup.getType()); - SimpleDateFormat formatterDateTime = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ"); - try { - strayBackup.setDate(formatterDateTime.parse(strayNetworkerBackup.getSaveTime())); - } catch (ParseException e) { - String msg = String.format("Unable to parse date [%s].", strayNetworkerBackup.getSaveTime()); - LOG.error(msg, e); - throw new CloudRuntimeException(msg, e); - } - strayBackup.setStatus(Backup.Status.BackedUp); - for ( Backup.VolumeInfo thisVMVol : vm.getBackupVolumeList()) { - vmBackupSize += (thisVMVol.getSize() / 1024L /1024L); - } - strayBackup.setSize(vmBackupSize); - strayBackup.setProtectedSize(strayNetworkerBackup.getSize().getValue() / 1024L ); - strayBackup.setBackupOfferingId(vm.getBackupOfferingId()); - strayBackup.setAccountId(vm.getAccountId()); - strayBackup.setDomainId(vm.getDomainId()); - strayBackup.setZoneId(vm.getDataCenterId()); - LOG.debug(String.format("Creating a new entry in backups: [id: %s, uuid: %s, vm_id: %s, external_id: %s, type: %s, date: %s, backup_offering_id: %s, account_id: %s, " - + "domain_id: %s, zone_id: %s].", strayBackup.getId(), strayBackup.getUuid(), strayBackup.getVmId(), strayBackup.getExternalId(), - strayBackup.getType(), strayBackup.getDate(), strayBackup.getBackupOfferingId(), strayBackup.getAccountId(), - strayBackup.getDomainId(), strayBackup.getZoneId())); - backupDao.persist(strayBackup); - LOG.warn("Added backup found in provider [" + strayBackup + "]"); - } else { - LOG.debug ("Backup is in progress, skipping addition for this run"); - } - } - for (final Long backupIdToRemove : removeList) { - LOG.warn(String.format("Removing backup with ID: [%s].", backupIdToRemove)); - backupDao.remove(backupIdToRemove); - } + public Backup createNewBackupEntryForRestorePoint(Backup.RestorePoint restorePoint, VirtualMachine vm, Backup.Metric metric) { + // Technically an administrator can manually create a backup for a VM by utilizing the KVM scripts + // with the proper parameters. So we will register any backups taken on the Networker side from + // outside Cloudstack. If ever Networker will support KVM out of the box this functionality also will + // ensure that SLA like backups will be found and registered. + NetworkerBackup strayNetworkerBackup = getClient(vm.getDataCenterId()).getNetworkerBackupInfo(restorePoint.getId()); + + // Since running backups are already present in Networker Server but not completed + // make sure the backup is not in progress at this time. + if (strayNetworkerBackup.getCompletionTime() != null) { + BackupVO backup = new BackupVO(); + backup.setVmId(vm.getId()); + backup.setExternalId(strayNetworkerBackup.getId()); + backup.setType(strayNetworkerBackup.getType()); + SimpleDateFormat formatterDateTime = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ"); + try { + backup.setDate(formatterDateTime.parse(strayNetworkerBackup.getSaveTime())); + } catch (ParseException e) { + String msg = String.format("Unable to parse date [%s].", strayNetworkerBackup.getSaveTime()); + LOG.error(msg, e); + throw new CloudRuntimeException(msg, e); } - }); + backup.setStatus(Backup.Status.BackedUp); + Long vmBackupProtectedSize=0L; + for (Backup.VolumeInfo thisVMVol : vm.getBackupVolumeList()) { + vmBackupProtectedSize += (thisVMVol.getSize() / 1024L / 1024L); + } + backup.setSize(strayNetworkerBackup.getSize().getValue() / 1024L); + backup.setProtectedSize(vmBackupProtectedSize); + backup.setBackupOfferingId(vm.getBackupOfferingId()); + backup.setAccountId(vm.getAccountId()); + backup.setDomainId(vm.getDomainId()); + backup.setZoneId(vm.getDataCenterId()); + backupDao.persist(backup); + return backup; + } + LOG.debug ("Backup is in progress, skipping addition for this run"); + return null; + } + + @Override + public List listRestorePoints(VirtualMachine vm) { + final Long zoneId = vm.getDataCenterId(); + final ArrayList backupIds = getClient(zoneId).getBackupsForVm(vm); + List restorePoints = + backupIds.stream().map(id -> new Backup.RestorePoint(id, null, null)).collect(Collectors.toList()); + return restorePoints; } @Override diff --git a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java index c120d8bd599..0735136d15d 100644 --- a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java +++ b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java @@ -29,8 +29,6 @@ import java.util.stream.Collectors; import javax.inject.Inject; -import org.apache.cloudstack.api.ApiCommandResourceType; -import org.apache.cloudstack.api.InternalIdentity; import org.apache.cloudstack.backup.Backup.Metric; import org.apache.cloudstack.backup.dao.BackupDao; import org.apache.cloudstack.backup.veeam.VeeamClient; @@ -42,20 +40,13 @@ import org.apache.commons.lang3.BooleanUtils; import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; -import com.cloud.event.ActionEventUtils; -import com.cloud.event.EventTypes; -import com.cloud.event.EventVO; import com.cloud.hypervisor.Hypervisor; import com.cloud.dc.VmwareDatacenter; import com.cloud.hypervisor.vmware.VmwareDatacenterZoneMap; import com.cloud.dc.dao.VmwareDatacenterDao; import com.cloud.hypervisor.vmware.dao.VmwareDatacenterZoneMapDao; -import com.cloud.user.User; import com.cloud.utils.Pair; import com.cloud.utils.component.AdapterBase; -import com.cloud.utils.db.Transaction; -import com.cloud.utils.db.TransactionCallbackNoReturn; -import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; @@ -220,9 +211,10 @@ public class VeeamBackupProvider extends AdapterBase implements BackupProvider, } @Override - public boolean takeBackup(final VirtualMachine vm) { + public Pair takeBackup(final VirtualMachine vm) { final VeeamClient client = getClient(vm.getDataCenterId()); - return client.startBackupJob(vm.getBackupExternalId()); + Boolean result = client.startBackupJob(vm.getBackupExternalId()); + return new Pair<>(result, null); } @Override @@ -322,78 +314,30 @@ public class VeeamBackupProvider extends AdapterBase implements BackupProvider, return metrics; } - private List listRestorePoints(VirtualMachine vm) { - String backupName = getGuestBackupName(vm.getInstanceName(), vm.getUuid()); - return getClient(vm.getDataCenterId()).listRestorePoints(backupName, vm.getInstanceName()); - } - - private Backup checkAndUpdateIfBackupEntryExistsForRestorePoint(List backupsInDb, Backup.RestorePoint restorePoint, Backup.Metric metric) { - for (final Backup backup : backupsInDb) { - if (restorePoint.getId().equals(backup.getExternalId())) { - if (metric != null) { - logger.debug("Update backup with [id: {}, uuid: {}, name: {}, external id: {}] from [size: {}, protected size: {}] to [size: {}, protected size: {}].", - backup.getId(), backup.getUuid(), backup.getName(), backup.getExternalId(), backup.getSize(), backup.getProtectedSize(), metric.getBackupSize(), metric.getDataSize()); - - ((BackupVO) backup).setSize(metric.getBackupSize()); - ((BackupVO) backup).setProtectedSize(metric.getDataSize()); - backupDao.update(backup.getId(), ((BackupVO) backup)); - } - return backup; - } + @Override + public Backup createNewBackupEntryForRestorePoint(Backup.RestorePoint restorePoint, VirtualMachine vm, Backup.Metric metric) { + BackupVO backup = new BackupVO(); + backup.setVmId(vm.getId()); + backup.setExternalId(restorePoint.getId()); + backup.setType(restorePoint.getType()); + backup.setDate(restorePoint.getCreated()); + backup.setStatus(Backup.Status.BackedUp); + if (metric != null) { + backup.setSize(metric.getBackupSize()); + backup.setProtectedSize(metric.getDataSize()); } - return null; + backup.setBackupOfferingId(vm.getBackupOfferingId()); + backup.setAccountId(vm.getAccountId()); + backup.setDomainId(vm.getDomainId()); + backup.setZoneId(vm.getDataCenterId()); + backupDao.persist(backup); + return backup; } @Override - public void syncBackups(VirtualMachine vm, Backup.Metric metric) { - List restorePoints = listRestorePoints(vm); - if (CollectionUtils.isEmpty(restorePoints)) { - logger.debug("Can't find any restore point to VM: {}", vm); - return; - } - Transaction.execute(new TransactionCallbackNoReturn() { - @Override - public void doInTransactionWithoutResult(TransactionStatus status) { - final List backupsInDb = backupDao.listByVmId(null, vm.getId()); - final List removeList = backupsInDb.stream().map(InternalIdentity::getId).collect(Collectors.toList()); - for (final Backup.RestorePoint restorePoint : restorePoints) { - if (!(restorePoint.getId() == null || restorePoint.getType() == null || restorePoint.getCreated() == null)) { - Backup existingBackupEntry = checkAndUpdateIfBackupEntryExistsForRestorePoint(backupsInDb, restorePoint, metric); - if (existingBackupEntry != null) { - removeList.remove(existingBackupEntry.getId()); - continue; - } - - BackupVO backup = new BackupVO(); - backup.setVmId(vm.getId()); - backup.setExternalId(restorePoint.getId()); - backup.setType(restorePoint.getType()); - backup.setDate(restorePoint.getCreated()); - backup.setStatus(Backup.Status.BackedUp); - if (metric != null) { - backup.setSize(metric.getBackupSize()); - backup.setProtectedSize(metric.getDataSize()); - } - backup.setBackupOfferingId(vm.getBackupOfferingId()); - backup.setAccountId(vm.getAccountId()); - backup.setDomainId(vm.getDomainId()); - backup.setZoneId(vm.getDataCenterId()); - - logger.debug("Creating a new entry in backups: [id: {}, uuid: {}, name: {}, vm_id: {}, external_id: {}, type: {}, date: {}, backup_offering_id: {}, account_id: {}, " - + "domain_id: {}, zone_id: {}].", backup.getId(), backup.getUuid(), backup.getName(), backup.getVmId(), backup.getExternalId(), backup.getType(), backup.getDate(), backup.getBackupOfferingId(), backup.getAccountId(), backup.getDomainId(), backup.getZoneId()); - backupDao.persist(backup); - - ActionEventUtils.onCompletedActionEvent(User.UID_SYSTEM, vm.getAccountId(), EventVO.LEVEL_INFO, EventTypes.EVENT_VM_BACKUP_CREATE, - String.format("Created backup %s for VM ID: %s", backup.getUuid(), vm.getUuid()), - vm.getId(), ApiCommandResourceType.VirtualMachine.toString(),0); - } - } - for (final Long backupIdToRemove : removeList) { - logger.warn(String.format("Removing backup with ID: [%s].", backupIdToRemove)); - backupDao.remove(backupIdToRemove); - } - } - }); + public List listRestorePoints(VirtualMachine vm) { + String backupName = getGuestBackupName(vm.getInstanceName(), vm.getUuid()); + return getClient(vm.getDataCenterId()).listRestorePoints(backupName, vm.getInstanceName()); } @Override diff --git a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java index d911736090c..9accc0714de 100644 --- a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java +++ b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java @@ -844,11 +844,11 @@ public class VeeamClient { "if ($restore) { $restore ^| Format-List } }" ); Pair response = executePowerShellCommands(cmds); - final List restorePoints = new ArrayList<>(); if (response == null || !response.first()) { - return restorePoints; + return null; } + final List restorePoints = new ArrayList<>(); for (final String block : response.second().split("\r\n\r\n")) { if (block.isEmpty()) { continue; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 2f57ad6ffb0..29374f3e594 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -3155,7 +3155,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv disk.setLogicalBlockIOSize(pool.getSupportedLogicalBlockSize()); disk.setPhysicalBlockIOSize(pool.getSupportedPhysicalBlockSize()); - if (diskBusType == DiskDef.DiskBus.SCSI ) { + if (diskBusType == DiskDef.DiskBus.SCSI || diskBusType == DiskDef.DiskBus.VIRTIOBLK) { disk.setQemuDriver(true); disk.setDiscard(DiscardType.UNMAP); } @@ -3226,7 +3226,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv disk.setCacheMode(DiskDef.DiskCacheMode.valueOf(volumeObjectTO.getCacheMode().toString().toUpperCase())); } - if (volumeObjectTO.requiresEncryption()) { + if (volumeObjectTO.requiresEncryption() && + pool.getType().encryptionSupportMode() == Storage.EncryptionSupport.Hypervisor ) { String secretUuid = createLibvirtVolumeSecret(conn, volumeObjectTO.getPath(), volumeObjectTO.getPassphrase()); DiskDef.LibvirtDiskEncryptDetails encryptDetails = new DiskDef.LibvirtDiskEncryptDetails(secretUuid, QemuObject.EncryptFormat.enumValue(volumeObjectTO.getEncryptFormat())); disk.setLibvirtDiskEncryptDetails(encryptDetails); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java index 82617696954..1c504b6239f 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java @@ -248,7 +248,7 @@ public class LibvirtVMDef { guestDef.append("\n"); } } - if (!(_arch != null && _arch.equals("s390x"))) { + if (_arch == null || ! (_arch.equals("aarch64") || _arch.equals("s390x"))) { // simplification of (as ref.) (!(_arch != null && _arch.equals("s390x")) || (_arch == null || !_arch.equals("aarch64"))) guestDef.append("\n"); } guestDef.append("\n"); @@ -680,7 +680,7 @@ public class LibvirtVMDef { } public enum DiskBus { - IDE("ide"), SCSI("scsi"), VIRTIO("virtio"), XEN("xen"), USB("usb"), UML("uml"), FDC("fdc"), SATA("sata"); + IDE("ide"), SCSI("scsi"), VIRTIO("virtio"), XEN("xen"), USB("usb"), UML("uml"), FDC("fdc"), SATA("sata"), VIRTIOBLK("virtio-blk"); String _bus; DiskBus(String bus) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetUnmanagedInstancesCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetUnmanagedInstancesCommandWrapper.java index 9495646bad5..70b1715cc20 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetUnmanagedInstancesCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetUnmanagedInstancesCommandWrapper.java @@ -122,7 +122,10 @@ public final class LibvirtGetUnmanagedInstancesCommandWrapper extends CommandWra instance.setName(domain.getName()); instance.setCpuCores((int) LibvirtComputingResource.countDomainRunningVcpus(domain)); - instance.setCpuSpeed(parser.getCpuTuneDef().getShares()/instance.getCpuCores()); + + if (parser.getCpuTuneDef() != null && instance.getCpuCores() != null) { + instance.setCpuSpeed(parser.getCpuTuneDef().getShares()/instance.getCpuCores()); + } if (parser.getCpuModeDef() != null) { instance.setCpuCoresPerSocket(parser.getCpuModeDef().getCoresPerSocket()); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index 92e4570170e..5e62671dd22 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -125,6 +125,7 @@ import com.cloud.hypervisor.kvm.resource.wrapper.LibvirtUtilitiesHelper; import com.cloud.storage.JavaStorageLayer; import com.cloud.storage.MigrationOptions; import com.cloud.storage.ScopeType; +import com.cloud.storage.Storage; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.Storage.StoragePoolType; import com.cloud.storage.StorageLayer; @@ -161,7 +162,7 @@ public class KVMStorageProcessor implements StorageProcessor { /** * Time interval before rechecking virsh commands */ - private long waitDelayForVirshCommands = 1000l; + private final long waitDelayForVirshCommands = 1000L; public KVMStorageProcessor(final KVMStoragePoolManager storagePoolMgr, final LibvirtComputingResource resource) { this.storagePoolMgr = storagePoolMgr; @@ -258,7 +259,7 @@ public class KVMStorageProcessor implements StorageProcessor { logger.debug("Copying template to primary storage, template format is " + tmplVol.getFormat() ); final KVMStoragePool primaryPool = storagePoolMgr.getStoragePool(primaryStore.getPoolType(), primaryStore.getUuid()); - KVMPhysicalDisk primaryVol = null; + KVMPhysicalDisk primaryVol; if (destData instanceof VolumeObjectTO) { final VolumeObjectTO volume = (VolumeObjectTO)destData; // pass along volume's target size if it's bigger than template's size, for storage types that copy template rather than cloning on deploy @@ -277,8 +278,13 @@ public class KVMStorageProcessor implements StorageProcessor { String path = derivePath(primaryStore, destData, details); - if (!storagePoolMgr.connectPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), path, details)) { + if (path == null) { + path = destTempl.getUuid(); + } + + if (path != null && !storagePoolMgr.connectPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), path, details)) { logger.warn("Failed to connect physical disk at path: {}, in storage pool [id: {}, name: {}]", path, primaryStore.getUuid(), primaryStore.getName()); + return new PrimaryStorageDownloadAnswer("Failed to spool template disk at path: " + path + ", in storage pool id: " + primaryStore.getUuid()); } primaryVol = storagePoolMgr.copyPhysicalDisk(tmplVol, path != null ? path : destTempl.getUuid(), primaryPool, cmd.getWaitInMillSeconds()); @@ -336,12 +342,13 @@ public class KVMStorageProcessor implements StorageProcessor { } private String derivePath(PrimaryDataStoreTO primaryStore, DataTO destData, Map details) { - String path = null; + String path; if (primaryStore.getPoolType() == StoragePoolType.FiberChannel) { path = destData.getPath(); } else { path = details != null ? details.get("managedStoreTarget") : null; } + return path; } @@ -390,8 +397,7 @@ public class KVMStorageProcessor implements StorageProcessor { logger.debug("Using templates disk size of " + toHumanReadableSize(templateVol.getVirtualSize()) + "since size passed was " + toHumanReadableSize(size)); } - final KVMPhysicalDisk primaryVol = storagePoolMgr.copyPhysicalDisk(templateVol, volUuid, primaryPool, timeout); - return primaryVol; + return storagePoolMgr.copyPhysicalDisk(templateVol, volUuid, primaryPool, timeout); } catch (final CloudRuntimeException e) { logger.error("Failed to download template to primary storage", e); return null; @@ -410,9 +416,9 @@ public class KVMStorageProcessor implements StorageProcessor { final DataStoreTO imageStore = template.getDataStore(); final VolumeObjectTO volume = (VolumeObjectTO)destData; final PrimaryDataStoreTO primaryStore = (PrimaryDataStoreTO)volume.getDataStore(); - KVMPhysicalDisk BaseVol = null; - KVMStoragePool primaryPool = null; - KVMPhysicalDisk vol = null; + KVMPhysicalDisk BaseVol; + KVMStoragePool primaryPool; + KVMPhysicalDisk vol; try { primaryPool = storagePoolMgr.getStoragePool(primaryStore.getPoolType(), primaryStore.getUuid()); @@ -420,7 +426,7 @@ public class KVMStorageProcessor implements StorageProcessor { String templatePath = template.getPath(); if (primaryPool.getType() == StoragePoolType.CLVM) { - templatePath = ((NfsTO)imageStore).getUrl() + File.separator + templatePath; + templatePath = imageStore.getUrl() + File.separator + templatePath; vol = templateToPrimaryDownload(templatePath, primaryPool, volume.getUuid(), volume.getSize(), cmd.getWaitInMillSeconds()); } if (storagePoolMgr.supportsPhysicalDiskCopy(primaryPool.getType())) { Map details = primaryStore.getDetails(); @@ -778,15 +784,19 @@ public class KVMStorageProcessor implements StorageProcessor { KVMStoragePool secondaryStorage = null; + String path = null; try { // look for options indicating an overridden path or IQN. Used when snapshots have to be // temporarily copied on the manaaged storage device before the actual copy to target object Map details = cmd.getOptions(); - String path = details != null ? details.get(DiskTO.PATH) : null; + path = details != null ? details.get(DiskTO.PATH) : null; if (path == null) { path = details != null ? details.get(DiskTO.IQN) : null; if (path == null) { - new CloudRuntimeException("The 'path' or 'iqn' field must be specified."); + path = srcData.getPath(); + if (path == null) { + new CloudRuntimeException("The 'path' or 'iqn' field must be specified."); + } } } @@ -849,8 +859,6 @@ public class KVMStorageProcessor implements StorageProcessor { loc.addFormat(info); loc.save(); - storagePoolMgr.disconnectPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), path); - TemplateObjectTO newTemplate = new TemplateObjectTO(); newTemplate.setPath(templateFolder + File.separator + templateName + ".qcow2"); @@ -870,6 +878,10 @@ public class KVMStorageProcessor implements StorageProcessor { return new CopyCmdAnswer(ex.toString()); } finally { + if (path != null) { + storagePoolMgr.disconnectPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), path); + } + if (secondaryStorage != null) { secondaryStorage.delete(); } @@ -1045,7 +1057,9 @@ public class KVMStorageProcessor implements StorageProcessor { command.add(NAME_OPTION, snapshotName); command.add("-p", snapshotDestPath); - descName = UUID.randomUUID().toString(); + if (isCreatedFromVmSnapshot) { + descName = UUID.randomUUID().toString(); + } command.add("-t", descName); final String result = command.execute(); @@ -1415,12 +1429,14 @@ public class KVMStorageProcessor implements StorageProcessor { if (disk.getDeviceType() == DeviceType.DISK) { if (disk.getBusType() == DiskDef.DiskBus.SCSI) { busT = DiskDef.DiskBus.SCSI; + } else if (disk.getBusType() == DiskDef.DiskBus.VIRTIOBLK) { + busT = DiskDef.DiskBus.VIRTIOBLK; } break; } } diskdef = new DiskDef(); - if (busT == DiskDef.DiskBus.SCSI) { + if (busT == DiskDef.DiskBus.SCSI || busT == DiskDef.DiskBus.VIRTIOBLK) { diskdef.setQemuDriver(true); diskdef.setDiscard(DiscardType.UNMAP); } @@ -1459,7 +1475,8 @@ public class KVMStorageProcessor implements StorageProcessor { } } - if (encryptDetails != null) { + if (encryptDetails != null && + attachingPool.getType().encryptionSupportMode() == Storage.EncryptionSupport.Hypervisor) { diskdef.setLibvirtDiskEncryptDetails(encryptDetails); } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIAdapterBase.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIAdapterBase.java index 0cf8ce0018d..7ba29ffc26e 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIAdapterBase.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIAdapterBase.java @@ -162,6 +162,13 @@ public abstract class MultipathSCSIAdapterBase implements StorageAdaptor { KVMPhysicalDisk disk = new KVMPhysicalDisk(address.getPath(), address.toString(), pool); disk.setFormat(QemuImg.PhysicalDiskFormat.RAW); + // validate we have a connection, if not we need to connect first. + if (!isConnected(address.getPath())) { + if (!connectPhysicalDisk(address, pool, null)) { + throw new CloudRuntimeException("Unable to connect to volume " + address.getPath()); + } + } + long diskSize = getPhysicalDiskSize(address.getPath()); disk.setSize(diskSize); disk.setVirtualSize(diskSize); @@ -199,6 +206,10 @@ public abstract class MultipathSCSIAdapterBase implements StorageAdaptor { // we expect WWN values in the volumePath so need to convert it to an actual physical path AddressInfo address = this.parseAndValidatePath(volumePath); + return connectPhysicalDisk(address, pool, details); + } + + private boolean connectPhysicalDisk(AddressInfo address, KVMStoragePool pool, Map details) { // validate we have a connection id - we can't proceed without that if (address.getConnectionId() == null) { LOGGER.error("Unable to connect volume with address [" + address.getPath() + "] of the storage pool: " + pool.getUuid() + " - connection id is not set in provided path"); @@ -510,6 +521,18 @@ public abstract class MultipathSCSIAdapterBase implements StorageAdaptor { return false; } + boolean isConnected(String path) { + // run a command to test if this is a binary device at this path + Script blockTest = new Script("/bin/test", LOGGER); + blockTest.add("-b", path); + blockTest.execute(); + int rc = blockTest.getExitValue(); + if (rc == 0) { + return true; + } + return false; + } + long getPhysicalDiskSize(String diskPath) { if (StringUtils.isEmpty(diskPath)) { return 0; diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java index fb5e85f8647..920b3a439d0 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java @@ -6531,4 +6531,14 @@ public class LibvirtComputingResourceTest { assertEquals(DiskDef.DiscardType.UNMAP, rootDisk.getDiscard()); } } + + @Test + public void testGetDiskModelFromVMDetailVirtioBlk() { + VirtualMachineTO virtualMachineTO = Mockito.mock(VirtualMachineTO.class); + Map details = new HashMap<>(); + details.put(VmDetailConstants.ROOT_DISK_CONTROLLER, "virtio-blk"); + Mockito.when(virtualMachineTO.getDetails()).thenReturn(details); + DiskDef.DiskBus diskBus = libvirtComputingResourceSpy.getDiskModelFromVMDetail(virtualMachineTO); + assertEquals(DiskDef.DiskBus.VIRTIOBLK, diskBus); + } } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java index b8bf08cd01a..c02513f4889 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java @@ -27,6 +27,7 @@ import java.util.List; import java.util.Map; import java.util.UUID; +import com.cloud.agent.api.CleanupVMCommand; import javax.inject.Inject; import com.cloud.agent.api.to.NfsTO; @@ -370,6 +371,13 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co return tokens[0] + "@" + vCenterIp; } + @Override public List finalizeExpunge(VirtualMachine vm) { + List commands = new ArrayList(); + final CleanupVMCommand cleanupVMCommand = new CleanupVMCommand(vm.getInstanceName(), true); + commands.add(cleanupVMCommand); + return commands; + } + @Override public List finalizeExpungeNics(VirtualMachine vm, List nics) { List commands = new ArrayList(); List nicVOs = nicDao.listByVmId(vm.getId()); diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenterService.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenterService.java index bbac78b879a..6bb473536b8 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenterService.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/VmwareDatacenterService.java @@ -22,12 +22,15 @@ import com.cloud.dc.VmwareDatacenterVO; import com.cloud.dc.VsphereStoragePolicy; import com.cloud.exception.DiscoveryException; import com.cloud.exception.ResourceInUseException; +import com.cloud.hypervisor.vmware.mo.HostMO; import com.cloud.storage.StoragePool; +import com.cloud.utils.Pair; import com.cloud.utils.component.PluggableService; import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.api.command.admin.zone.AddVmwareDcCmd; import org.apache.cloudstack.api.command.admin.zone.ImportVsphereStoragePoliciesCmd; import org.apache.cloudstack.api.command.admin.zone.ListVmwareDcVmsCmd; +import org.apache.cloudstack.api.command.admin.zone.ListVmwareDcHostsCmd; import org.apache.cloudstack.api.command.admin.zone.ListVmwareDcsCmd; import org.apache.cloudstack.api.command.admin.zone.ListVsphereStoragePoliciesCmd; import org.apache.cloudstack.api.command.admin.zone.ListVsphereStoragePolicyCompatiblePoolsCmd; @@ -53,5 +56,7 @@ public interface VmwareDatacenterService extends PluggableService { List listVsphereStoragePolicyCompatibleStoragePools(ListVsphereStoragePolicyCompatiblePoolsCmd cmd); - List listVMsInDatacenter(ListVmwareDcVmsCmd cmd); + List listHostsInDatacenter(ListVmwareDcHostsCmd cmd); + + Pair> listVMsInDatacenter(ListVmwareDcVmsCmd cmd); } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java index 561034155c2..b64422482aa 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java @@ -19,10 +19,12 @@ package com.cloud.hypervisor.vmware.manager; import java.io.File; import java.io.FileInputStream; import java.io.IOException; +import java.lang.reflect.InvocationTargetException; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; import java.rmi.RemoteException; import java.time.Duration; import java.time.Instant; @@ -43,10 +45,11 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; import javax.persistence.EntityExistsException; -import com.cloud.hypervisor.vmware.util.VmwareClient; import org.apache.cloudstack.api.command.admin.zone.AddVmwareDcCmd; import org.apache.cloudstack.api.command.admin.zone.ImportVsphereStoragePoliciesCmd; import org.apache.cloudstack.api.command.admin.zone.ListVmwareDcVmsCmd; +import org.apache.cloudstack.api.command.admin.zone.ListVmwareDcHostsCmd; +import org.apache.cloudstack.api.command.admin.zone.ListVmwareDcItems; import org.apache.cloudstack.api.command.admin.zone.ListVmwareDcsCmd; import org.apache.cloudstack.api.command.admin.zone.ListVsphereStoragePoliciesCmd; import org.apache.cloudstack.api.command.admin.zone.ListVsphereStoragePolicyCompatiblePoolsCmd; @@ -86,6 +89,7 @@ import com.cloud.dc.ClusterDetailsDao; import com.cloud.dc.ClusterVO; import com.cloud.dc.ClusterVSMMapVO; import com.cloud.dc.DataCenterVO; +import com.cloud.dc.VmwareDatacenter; import com.cloud.dc.VsphereStoragePolicy; import com.cloud.dc.VsphereStoragePolicyVO; import com.cloud.dc.dao.ClusterDao; @@ -111,7 +115,8 @@ import com.cloud.hypervisor.HypervisorGuruManager; import com.cloud.hypervisor.dao.HypervisorCapabilitiesDao; import com.cloud.hypervisor.vmware.LegacyZoneVO; import com.cloud.hypervisor.vmware.VmwareCleanupMaid; -import com.cloud.dc.VmwareDatacenter; +import com.cloud.hypervisor.vmware.util.VmwareClient; +import com.cloud.hypervisor.vmware.util.VmwareClientException; import com.cloud.hypervisor.vmware.VmwareDatacenterService; import com.cloud.dc.VmwareDatacenterVO; import com.cloud.hypervisor.vmware.VmwareDatacenterZoneMap; @@ -168,9 +173,16 @@ import com.cloud.utils.ssh.SshHelper; import com.cloud.vm.DomainRouterVO; import com.cloud.vm.dao.UserVmCloneSettingDao; import com.cloud.vm.dao.VMInstanceDao; + +// TODO move these items upstream? import com.vmware.pbm.PbmProfile; import com.vmware.vim25.AboutInfo; import com.vmware.vim25.ManagedObjectReference; +import com.vmware.vim25.InvalidLocaleFaultMsg; +import com.vmware.vim25.InvalidLoginFaultMsg; +import com.vmware.vim25.RuntimeFaultFaultMsg; +import com.vmware.vim25.InvalidPropertyFaultMsg; +import org.jetbrains.annotations.NotNull; public class VmwareManagerImpl extends ManagerBase implements VmwareManager, VmwareStorageMount, Listener, VmwareDatacenterService, Configurable { @@ -245,11 +257,11 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw private StorageLayer _storage; private final String _privateNetworkVSwitchName = "vSwitch0"; - private int _portsPerDvPortGroup = DEFAULT_PORTS_PER_DV_PORT_GROUP; + private final int _portsPerDvPortGroup = DEFAULT_PORTS_PER_DV_PORT_GROUP; private boolean _fullCloneFlag; private boolean _instanceNameFlag; private String _serviceConsoleName; - private String _managemetPortGroupName; + private String _managementPortGroupName; private String _defaultSystemVmNicAdapterType = VirtualEthernetCardType.E1000.toString(); private String _recycleHungWorker = "false"; private int _additionalPortRangeStart; @@ -263,7 +275,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw private final Random _rand = new Random(System.currentTimeMillis()); - private static ScheduledExecutorService templateCleanupScheduler = Executors.newScheduledThreadPool(1, new NamedThreadFactory("Vmware-FullyClonedTemplateCheck"));; + private static final ScheduledExecutorService templateCleanupScheduler = Executors.newScheduledThreadPool(1, new NamedThreadFactory("Vmware-FullyClonedTemplateCheck")); private final VmwareStorageManager _storageMgr; private final GlobalLock _exclusiveOpLock = GlobalLock.getInternLock("vmware.exclusive.op"); @@ -347,9 +359,9 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw _serviceConsoleName = "Service Console"; } - _managemetPortGroupName = _configDao.getValue(Config.VmwareManagementPortGroup.key()); - if (_managemetPortGroupName == null) { - _managemetPortGroupName = "Management Network"; + _managementPortGroupName = _configDao.getValue(Config.VmwareManagementPortGroup.key()); + if (_managementPortGroupName == null) { + _managementPortGroupName = "Management Network"; } _defaultSystemVmNicAdapterType = _configDao.getValue(Config.VmwareSystemVmNicDeviceType.key()); @@ -448,7 +460,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw logger.info("Preparing network on host " + hostMo.getContext().toString() + " for " + privateTrafficLabel); VirtualSwitchType vsType = VirtualSwitchType.getType(vSwitchType); - //The management network is probably always going to be a physical network with islation type of vlans, so assume BroadcastDomainType VLAN + //The management network is probably always going to be a physical network with isolation type of vlans, so assume BroadcastDomainType VLAN if (VirtualSwitchType.StandardVirtualSwitch == vsType) { HypervisorHostHelper.prepareNetwork(vSwitchName, "cloud.private", hostMo, vlanId, null, null, 180000, false, BroadcastDomainType.Vlan, null, null); } @@ -457,7 +469,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw AboutInfo about = hostMo.getHostAboutInfo(); if (about != null) { String version = about.getApiVersion(); - if (version != null && (version.equals("4.0") || version.equals("4.1")) && _portsPerDvPortGroup < DEFAULT_PORTS_PER_DV_PORT_GROUP_VSPHERE4_x) { + if (version != null && (version.equals("4.0") || version.equals("4.1")) ) { // && _portsPerDvPortGroup < DEFAULT_PORTS_PER_DV_PORT_GROUP_VSPHERE4_x) portsPerDvPortGroup = DEFAULT_PORTS_PER_DV_PORT_GROUP_VSPHERE4_x; } } @@ -480,7 +492,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw } URI uriForHost = new URI(UriUtils.encodeURIComponent(clusterDetails.get("url") + "/" + host.getName())); - morSrcHost = serviceContext.getHostMorByPath(URLDecoder.decode(uriForHost.getPath(), "UTF-8")); + morSrcHost = serviceContext.getHostMorByPath(URLDecoder.decode(uriForHost.getPath(), StandardCharsets.UTF_8)); if (morSrcHost == null) { return null; } @@ -496,19 +508,18 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw throw new CloudRuntimeException("Invalid serviceContext"); } ManagedObjectReference mor = serviceContext.getHostMorByPath(hostInventoryPath); - String privateTrafficLabel = null; + String privateTrafficLabel; privateTrafficLabel = serviceContext.getStockObject("privateTrafficLabel"); if (privateTrafficLabel == null) { privateTrafficLabel = _privateNetworkVSwitchName; } if (mor != null) { - List returnedHostList = new ArrayList(); + List returnedHostList = new ArrayList<>(); if (mor.getType().equals("ComputeResource")) { List hosts = serviceContext.getVimClient().getDynamicProperty(mor, "host"); - assert (hosts != null && hosts.size() > 0); - + assert (CollectionUtils.isNullOrEmpty(hosts)); // For ESX host, we need to enable host firewall to allow VNC access HostMO hostMo = new HostMO(serviceContext, hosts.get(0)); @@ -519,8 +530,8 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw List hosts = serviceContext.getVimClient().getDynamicProperty(mor, "host"); assert (hosts != null); - if (hosts.size() > 0) { - AboutInfo about = (AboutInfo)(serviceContext.getVimClient().getDynamicProperty(hosts.get(0), "config.product")); + if (!hosts.isEmpty()) { + AboutInfo about = serviceContext.getVimClient().getDynamicProperty(hosts.get(0), "config.product"); String version = about.getApiVersion(); int maxHostsPerCluster = _hvCapabilitiesDao.getMaxHostsPerCluster(HypervisorType.VMware, version); if (hosts.size() > maxHostsPerCluster) { @@ -549,7 +560,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw returnedHostList.add(mor); return returnedHostList; } else { - logger.error("Unsupport host type " + mor.getType() + ":" + mor.getValue() + " from inventory path: " + hostInventoryPath); + logger.error("Unsupport host type {}:{} from inventory path: {}", mor.getType(), mor.getValue(), hostInventoryPath); return null; } } @@ -614,13 +625,13 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw @Override public String getManagementPortGroupName() { - return _managemetPortGroupName; + return _managementPortGroupName; } @Override public String getManagementPortGroupByHost(HostMO hostMo) throws Exception { if (hostMo.getHostType() == VmwareHostType.ESXi) { - return _managemetPortGroupName; + return _managementPortGroupName; } return _serviceConsoleName; } @@ -630,7 +641,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw params.put("vmware.create.full.clone", _fullCloneFlag); params.put("vm.instancename.flag", _instanceNameFlag); params.put("service.console.name", _serviceConsoleName); - params.put("management.portgroup.name", _managemetPortGroupName); + params.put("management.portgroup.name", _managementPortGroupName); params.put("vmware.root.disk.controller", _rootDiskController); params.put("vmware.data.disk.controller", _dataDiskController); params.put("vmware.recycle.hung.wokervm", _recycleHungWorker); @@ -657,23 +668,23 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw return false; } - String tokens[] = workerTag.split("-"); + String[] tokens = workerTag.split("-"); if (tokens.length != 3) { logger.error("Invalid worker VM tag " + workerTag); return false; } long startTick = Long.parseLong(tokens[0]); - long msid = Long.parseLong(tokens[1]); - long runid = Long.parseLong(tokens[2]); + long msId = Long.parseLong(tokens[1]); + long runId = Long.parseLong(tokens[2]); - if (msHostPeerDao.countStateSeenInPeers(msid, runid, ManagementServerHost.State.Down) > 0) { + if (msHostPeerDao.countStateSeenInPeers(msId, runId, ManagementServerHost.State.Down) > 0) { if (logger.isInfoEnabled()) logger.info("Worker VM's owner management server node has been detected down from peer nodes, recycle it"); return true; } - if (runid != clusterManager.getManagementRunId(msid)) { + if (runId != clusterManager.getManagementRunId(msId)) { if (logger.isInfoEnabled()) logger.info("Worker VM's owner management server has changed runid, recycle it"); return true; @@ -710,7 +721,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw File patchFolder = new File(mountPoint + "/systemvm"); if (!patchFolder.exists()) { if (!patchFolder.mkdirs()) { - String msg = "Unable to create systemvm folder on secondary storage. location: " + patchFolder.toString(); + String msg = "Unable to create systemvm folder on secondary storage. location: " + patchFolder; logger.error(msg); throw new CloudRuntimeException(msg); } @@ -729,7 +740,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw } catch (IOException e) { logger.error("Unexpected exception ", e); - String msg = "Unable to copy systemvm ISO on secondary storage. src location: " + srcIso.toString() + ", dest location: " + destIso; + String msg = "Unable to copy systemvm ISO on secondary storage. src location: " + srcIso + ", dest location: " + destIso; logger.error(msg); throw new CloudRuntimeException(msg); } @@ -771,9 +782,8 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw isoFile = new File("/usr/share/cloudstack-common/vms/systemvm.iso"); } - assert (isoFile != null); if (!isoFile.exists()) { - logger.error("Unable to locate systemvm.iso in your setup at " + isoFile.toString()); + logger.error("Unable to locate systemvm.iso in your setup at " + isoFile); } return isoFile; } @@ -788,16 +798,16 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw if (keyFile == null || !keyFile.exists()) { keyFile = new File("/usr/share/cloudstack-common/scripts/vm/systemvm/id_rsa.cloud"); } - assert (keyFile != null); + if (!keyFile.exists()) { - logger.error("Unable to locate id_rsa.cloud in your setup at " + keyFile.toString()); + logger.error("Unable to locate id_rsa.cloud in your setup at " + keyFile); } return keyFile; } @Override public String getMountPoint(String storageUrl, String nfsVersion) { - String mountPoint = null; + String mountPoint; synchronized (_storageMounts) { mountPoint = _storageMounts.get(storageUrl); if (mountPoint != null) { @@ -827,7 +837,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw String mountPoint = null; long mshostId = ManagementServerNode.getManagementServerId(); for (int i = 0; i < 10; i++) { - String mntPt = parent + File.separator + String.valueOf(mshostId) + "." + Integer.toHexString(_rand.nextInt(Integer.MAX_VALUE)); + String mntPt = parent + File.separator + mshostId + "." + Integer.toHexString(_rand.nextInt(Integer.MAX_VALUE)); File file = new File(mntPt); if (!file.exists()) { if (_storage.mkdir(mntPt)) { @@ -852,10 +862,9 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw for (String mountPoint : mounts) { logger.info("umount NFS mount from previous session: " + mountPoint); - String result = null; Script command = new Script(true, "umount", _timeout, logger); command.add(mountPoint); - result = command.execute(); + String result = command.execute(); if (result != null) { logger.warn("Unable to umount " + mountPoint + " due to " + result); } @@ -873,10 +882,9 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw for (String mountPoint : _storageMounts.values()) { logger.info("umount NFS mount: " + mountPoint); - String result = null; Script command = new Script(true, "umount", _timeout, logger); command.add(mountPoint); - result = command.execute(); + String result = command.execute(); if (result != null) { logger.warn("Unable to umount " + mountPoint + " due to " + result); } @@ -894,8 +902,8 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw return null; } - Script script = null; - String result = null; + Script script; + String result; Script command = new Script(true, "mount", _timeout, logger); command.add("-t", "nfs"); if (nfsVersion != null){ @@ -982,11 +990,9 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw @Override public void processConnect(Host host, StartupCommand cmd, boolean forRebalance) { - if (cmd instanceof StartupCommand) { + if (cmd != null) { if (host.getHypervisorType() == HypervisorType.VMware) { updateClusterNativeHAState(host, cmd); - } else { - return; } } } @@ -1056,16 +1062,16 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw @Override public Pair getAddiionalVncPortRange() { - return new Pair(_additionalPortRangeStart, _additionalPortRangeSize); + return new Pair<>(_additionalPortRangeStart, _additionalPortRangeSize); } @Override public Map getNexusVSMCredentialsByClusterId(Long clusterId) { - CiscoNexusVSMDeviceVO nexusVSM = null; - ClusterVSMMapVO vsmMapVO = null; + CiscoNexusVSMDeviceVO nexusVSM; + ClusterVSMMapVO vsmMapVO; vsmMapVO = _vsmMapDao.findByClusterId(clusterId); - long vsmId = 0; + long vsmId; if (vsmMapVO != null) { vsmId = vsmMapVO.getVsmId(); logger.info("vsmId is " + vsmId); @@ -1076,7 +1082,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw return null; } - Map nexusVSMCredentials = new HashMap(); + Map nexusVSMCredentials = new HashMap<>(); if (nexusVSM != null) { nexusVSMCredentials.put("vsmip", nexusVSM.getipaddr()); nexusVSMCredentials.put("vsmusername", nexusVSM.getUserName()); @@ -1103,7 +1109,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw @Override public List> getCommands() { - List> cmdList = new ArrayList>(); + List> cmdList = new ArrayList<>(); cmdList.add(AddVmwareDcCmd.class); cmdList.add(UpdateVmwareDcCmd.class); cmdList.add(RemoveVmwareDcCmd.class); @@ -1112,13 +1118,14 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw cmdList.add(ListVsphereStoragePoliciesCmd.class); cmdList.add(ListVsphereStoragePolicyCompatiblePoolsCmd.class); cmdList.add(ListVmwareDcVmsCmd.class); + cmdList.add(ListVmwareDcHostsCmd.class); return cmdList; } @Override @DB public VmwareDatacenterVO addVmwareDatacenter(AddVmwareDcCmd cmd) throws ResourceInUseException { - VmwareDatacenterVO vmwareDc = null; + VmwareDatacenterVO vmwareDc; Long zoneId = cmd.getZoneId(); String userName = cmd.getUsername(); String password = cmd.getPassword(); @@ -1174,10 +1181,10 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw checkIfDcIsUsed(vCenterHost, vmwareDcName, zoneId); VmwareContext context = null; - DatacenterMO dcMo = null; + DatacenterMO dcMo; String dcCustomFieldValue; boolean addDcCustomFieldDef = false; - boolean dcInUse = false; + boolean dcInUse; String guid; ManagedObjectReference dcMor; try { @@ -1210,7 +1217,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw // Map zone with vmware datacenter vmwareDcZoneMap = new VmwareDatacenterZoneMapVO(zoneId, vmwareDc.getId()); - vmwareDcZoneMap = vmwareDatacenterZoneMapDao.persist(vmwareDcZoneMap); + vmwareDatacenterZoneMapDao.persist(vmwareDcZoneMap); // Set custom field for this DC if (addDcCustomFieldDef) { @@ -1230,7 +1237,6 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw if (context != null) { context.close(); } - context = null; } importVsphereStoragePoliciesInternal(zoneId, vmwareDc.getId()); return vmwareDc; @@ -1255,9 +1261,9 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw * Check if DC is already part of zone * In that case vmware_data_center table should have the DC and a dc zone mapping should exist * - * @param vCenterHost - * @param vmwareDcName - * @param zoneId + * @param vCenterHost the vcenter appliance hostname + * @param vmwareDcName the name of the vmware DC + * @param zoneId zone that the DC should be connected to * @throws ResourceInUseException if the DC can not be used. */ private void checkIfDcIsUsed(String vCenterHost, String vmwareDcName, Long zoneId) throws ResourceInUseException { @@ -1265,7 +1271,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw vmwareDc = vmwareDcDao.getVmwareDatacenterByGuid(vmwareDcName + "@" + vCenterHost); if (vmwareDc != null) { VmwareDatacenterZoneMapVO mapping = vmwareDatacenterZoneMapDao.findByVmwareDcId(vmwareDc.getId()); - if (mapping != null && Long.compare(zoneId, mapping.getZoneId()) == 0) { + if (mapping != null && zoneId == mapping.getZoneId()) { throw new ResourceInUseException(String.format("This DC (%s) is already part of other CloudStack zone (%d). Cannot add this DC to more zones.", vmwareDc.getUuid(), zoneId)); } } @@ -1274,7 +1280,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw @Override @ActionEvent(eventType = EventTypes.EVENT_ZONE_EDIT, eventDescription = "updating VMware datacenter") public VmwareDatacenter updateVmwareDatacenter(UpdateVmwareDcCmd cmd) { - final Long zoneId = cmd.getZoneId(); + final long zoneId = cmd.getZoneId(); final String userName = cmd.getUsername(); final String password = cmd.getPassword(); final String vCenterHost = cmd.getVcenter(); @@ -1302,7 +1308,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw } vmwareDc.setGuid(String.format("%s@%s", vmwareDc.getVmwareDatacenterName(), vmwareDc.getVcenterHost())); - return Transaction.execute(new TransactionCallback() { + return Transaction.execute(new TransactionCallback<>() { @Override public VmwareDatacenter doInTransaction(TransactionStatus status) { if (vmwareDcDao.update(vmwareDc.getId(), vmwareDc)) { @@ -1351,7 +1357,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw String vCenterHost; String userName; String password; - DatacenterMO dcMo = null; + DatacenterMO dcMo; final VmwareDatacenterZoneMapVO vmwareDcZoneMap = vmwareDatacenterZoneMapDao.findByZoneId(zoneId); // Check if zone is associated with VMware DC if (vmwareDcZoneMap == null) { @@ -1388,11 +1394,9 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw throw new DiscoveryException(msg); } - assert (dcMo != null); - // Reset custom field property cloud.zone over this DC dcMo.setCustomFieldValue(CustomFieldConstants.CLOUD_ZONE, "false"); - logger.info("Sucessfully reset custom field property cloud.zone over DC " + vmwareDcName); + logger.info("Sucessfully reset custom field property cloud.zone over DC {}", vmwareDcName); } catch (Exception e) { String msg = "Unable to reset custom field property cloud.zone over DC " + vmwareDcName + " due to : " + VmwareHelper.getExceptionMessage(e); logger.error(msg); @@ -1401,7 +1405,6 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw if (context != null) { context.close(); } - context = null; } return true; } @@ -1422,7 +1425,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw private void validateZoneWithResources(Long zoneId, String errStr) throws ResourceInUseException { // Check if zone has resources? - For now look for clusters List clusters = clusterDao.listByZoneId(zoneId); - if (clusters != null && clusters.size() > 0) { + if (!CollectionUtils.isNullOrEmpty(clusters)) { // Look for VMware hypervisor. for (ClusterVO cluster : clusters) { if (cluster.getHypervisorType().equals(HypervisorType.VMware)) { @@ -1443,9 +1446,9 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw } @Override - public List listVmwareDatacenters(ListVmwareDcsCmd cmd) throws CloudRuntimeException, InvalidParameterValueException { + public List listVmwareDatacenters(ListVmwareDcsCmd cmd) throws CloudRuntimeException { Long zoneId = cmd.getZoneId(); - List vmwareDcList = new ArrayList(); + List vmwareDcList = new ArrayList<>(); VmwareDatacenterZoneMapVO vmwareDcZoneMap; VmwareDatacenterVO vmwareDatacenter; long vmwareDcId; @@ -1503,7 +1506,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw String vCenterHost = vmwareDatacenter.getVcenterHost(); String userName = vmwareDatacenter.getUser(); String password = vmwareDatacenter.getPassword(); - List storageProfiles = null; + List storageProfiles; try { logger.debug(String.format("Importing vSphere Storage Policies for the vmware DC %d in zone %d", vmwareDcId, zoneId)); VmwareContext context = VmwareContextFactory.getContext(vCenterHost, userName, password); @@ -1531,16 +1534,15 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw List allStoragePolicies = vsphereStoragePolicyDao.listAll(); List finalStorageProfiles = storageProfiles; List needToMarkRemoved = allStoragePolicies.stream() - .filter(existingPolicy -> !finalStorageProfiles.stream() - .anyMatch(storageProfile -> storageProfile.getProfileId().getUniqueId().equals(existingPolicy.getPolicyId()))) + .filter(existingPolicy -> finalStorageProfiles.stream() + .noneMatch(storageProfile -> storageProfile.getProfileId().getUniqueId().equals(existingPolicy.getPolicyId()))) .collect(Collectors.toList()); for (VsphereStoragePolicyVO storagePolicy : needToMarkRemoved) { vsphereStoragePolicyDao.remove(storagePolicy.getId()); } - List storagePolicies = vsphereStoragePolicyDao.listAll(); - return storagePolicies; + return vsphereStoragePolicyDao.listAll(); } @Override @@ -1586,13 +1588,87 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw } @Override - public List listVMsInDatacenter(ListVmwareDcVmsCmd cmd) { + public List listHostsInDatacenter(ListVmwareDcHostsCmd cmd) { + VcenterData vmwaredc = getVcenterData(cmd); + + try { + VmwareContext context = getVmwareContext(vmwaredc); + DatacenterMO dcMo = getDatacenterMO(context, vmwaredc); + return dcMo.getAllHostsOnDatacenter(); + } catch (RuntimeFaultFaultMsg | URISyntaxException | VmwareClientException | InvalidLocaleFaultMsg | + InvalidLoginFaultMsg | InvalidPropertyFaultMsg e) { + String errorMsg = String.format("Error retrieving stopped VMs from the VMware VC %s datacenter %s: %s", + vmwaredc.vcenter, vmwaredc.datacenterName, e.getMessage()); + logger.error(errorMsg, e); + throw new CloudRuntimeException(errorMsg); + } + + } + + @Override + public Pair> listVMsInDatacenter(ListVmwareDcVmsCmd cmd) { + Integer maxObjects = cmd.getBatchSize(); + String token = cmd.getToken(); + String host = cmd.getHost(); + + VcenterData vmwaredc = getVcenterData(cmd); + + try { + VmwareContext context = getVmwareContext(vmwaredc); + + DatacenterMO dcMo = getDatacenterMO(context, vmwaredc); + + if (com.cloud.utils.StringUtils.isNotBlank(host)) { + ManagedObjectReference hostMor = dcMo.findHost(host); + if (hostMor == null) { + throw new VmwareClientException(String.format("No host '%s' found on DC: %s.", host, dcMo.getName())); + } + HostMO hostMo = new HostMO(context, hostMor); + return hostMo.getVms(maxObjects, token); + } else { + return dcMo.getVms(maxObjects, token); + } + } catch (InvalidParameterValueException | VmwareClientException | InvalidLocaleFaultMsg | InvalidLoginFaultMsg | + RuntimeFaultFaultMsg | URISyntaxException | InvalidPropertyFaultMsg | InvocationTargetException | + NoSuchMethodException | IllegalAccessException e) { + String errorMsg = String.format("Error retrieving stopped VMs from the VMware VC %s datacenter %s: %s", + vmwaredc.vcenter, vmwaredc.datacenterName, e.getMessage()); + logger.error(errorMsg, e); + throw new CloudRuntimeException(errorMsg); + } + } + + @NotNull + private DatacenterMO getDatacenterMO(VmwareContext context, VcenterData vmwaredc) throws InvalidPropertyFaultMsg, RuntimeFaultFaultMsg { + DatacenterMO dcMo = new DatacenterMO(context, vmwaredc.datacenterName); + ManagedObjectReference dcMor = dcMo.getMor(); + if (dcMor == null) { + String msg = String.format("Unable to find VMware datacenter %s in vCenter %s", + vmwaredc.datacenterName, vmwaredc.vcenter); + logger.error(msg); + throw new InvalidParameterValueException(msg); + } + return dcMo; + } + + @NotNull + private VmwareContext getVmwareContext(VcenterData vmwaredc) throws RuntimeFaultFaultMsg, URISyntaxException, VmwareClientException, InvalidLocaleFaultMsg, InvalidLoginFaultMsg { + logger.debug(String.format("Connecting to the VMware datacenter %s at vCenter %s to retrieve VMs", + vmwaredc.datacenterName, vmwaredc.vcenter)); + String serviceUrl = String.format("https://%s/sdk/vimService", vmwaredc.vcenter); + VmwareClient vimClient = new VmwareClient(vmwaredc.vcenter); + vimClient.connect(serviceUrl, vmwaredc.username, vmwaredc.password); + VmwareContext context = new VmwareContext(vimClient, vmwaredc.vcenter); + return context; + } + + @NotNull + private VcenterData getVcenterData(ListVmwareDcItems cmd) { String vcenter = cmd.getVcenter(); String datacenterName = cmd.getDatacenterName(); String username = cmd.getUsername(); String password = cmd.getPassword(); Long existingVcenterId = cmd.getExistingVcenterId(); - String keyword = cmd.getKeyword(); if ((existingVcenterId == null && StringUtils.isBlank(vcenter)) || (existingVcenterId != null && StringUtils.isNotBlank(vcenter))) { @@ -1613,37 +1689,27 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw username = vmwareDc.getUser(); password = vmwareDc.getPassword(); } + VcenterData vmwaredc = new VcenterData(vcenter, datacenterName, username, password); + return vmwaredc; + } - try { - logger.debug(String.format("Connecting to the VMware datacenter %s at vCenter %s to retrieve VMs", - datacenterName, vcenter)); - String serviceUrl = String.format("https://%s/sdk/vimService", vcenter); - VmwareClient vimClient = new VmwareClient(vcenter); - vimClient.connect(serviceUrl, username, password); - VmwareContext context = new VmwareContext(vimClient, vcenter); + private static class VcenterData { + public final String vcenter; + public final String datacenterName; + public final String username; + public final String password; - DatacenterMO dcMo = new DatacenterMO(context, datacenterName); - ManagedObjectReference dcMor = dcMo.getMor(); - if (dcMor == null) { - String msg = String.format("Unable to find VMware datacenter %s in vCenter %s", - datacenterName, vcenter); - logger.error(msg); - throw new InvalidParameterValueException(msg); - } - List instances = dcMo.getAllVmsOnDatacenter(); - return StringUtils.isBlank(keyword) ? instances : - instances.stream().filter(x -> x.getName().toLowerCase().contains(keyword.toLowerCase())).collect(Collectors.toList()); - } catch (Exception e) { - String errorMsg = String.format("Error retrieving stopped VMs from the VMware VC %s datacenter %s: %s", - vcenter, datacenterName, e.getMessage()); - logger.error(errorMsg, e); - throw new CloudRuntimeException(errorMsg); + public VcenterData(String vcenter, String datacenterName, String username, String password) { + this.vcenter = vcenter; + this.datacenterName = datacenterName; + this.username = username; + this.password = password; } } @Override public boolean hasNexusVSM(Long clusterId) { - ClusterVSMMapVO vsmMapVo = null; + ClusterVSMMapVO vsmMapVo; vsmMapVo = _vsmMapDao.findByClusterId(clusterId); if (vsmMapVo == null) { @@ -1693,7 +1759,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw } /** - * This task is to cleanup templates from primary storage that are otherwise not cleaned by the {@link com.cloud.storage.StorageManagerImpl.StorageGarbageCollector}. + * This task is to clean-up templates from primary storage that are otherwise not cleaned by the {@see com.cloud.storage.StorageManagerImpl.StorageGarbageCollector}. * it is called at regular intervals when storage.template.cleanup.enabled == true * It collect all templates that * - are deleted from cloudstack diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index f2c5ba2228e..a8ba033f3cc 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -45,6 +45,7 @@ import java.util.TimeZone; import java.util.UUID; import java.util.stream.Collectors; +import com.cloud.agent.api.CleanupVMCommand; import javax.naming.ConfigurationException; import javax.xml.datatype.XMLGregorianCalendar; @@ -583,6 +584,8 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes return execute((ResizeVolumeCommand) cmd); } else if (clz == UnregisterVMCommand.class) { return execute((UnregisterVMCommand) cmd); + } else if (clz == CleanupVMCommand.class) { + return execute((CleanupVMCommand) cmd); } else if (cmd instanceof StorageSubSystemCommand) { checkStorageProcessorAndHandlerNfsVersionAttribute((StorageSubSystemCommand) cmd); return storageHandler.handleStorageCommands((StorageSubSystemCommand) cmd); @@ -5796,6 +5799,26 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes return new Answer(cmd, true, "success"); } + protected Answer execute(CleanupVMCommand cmd) { + VmwareContext context = getServiceContext(); + VmwareHypervisorHost hyperHost = getHyperHost(context); + + try { + VirtualMachineMO vmMo = hyperHost.findVmOnHyperHost(cmd.getVmName()); + if (vmMo == null) { + String msg = String.format("VM [%s] not found on vCenter, cleanup not needed.", cmd.getVmName()); + logger.debug(msg); + return new Answer(cmd, true, msg); + } + vmMo.destroy(); + String msg = String.format("VM [%s] remnants on vCenter cleaned up.", cmd.getVmName()); + logger.debug(msg); + return new Answer(cmd, true, msg); + } catch (Exception e) { + return new Answer(cmd, false, createLogMessageException(e, cmd)); + } + } + protected Answer execute(UnregisterVMCommand cmd) { VmwareContext context = getServiceContext(); VmwareHypervisorHost hyperHost = getHyperHost(context); diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/AddVmwareDcCmd.java b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/AddVmwareDcCmd.java index 6f783e0a0fd..aa2e5ac6f3a 100644 --- a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/AddVmwareDcCmd.java +++ b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/AddVmwareDcCmd.java @@ -36,8 +36,8 @@ import com.cloud.dc.VmwareDatacenterVO; import com.cloud.user.Account; import com.cloud.utils.exception.CloudRuntimeException; -@APICommand(name = "addVmwareDc", description = "Adds a VMware datacenter to specified zone", responseObject = VmwareDatacenterResponse.class, - requestHasSensitiveInfo = true, responseHasSensitiveInfo = false) +@APICommand(name = "addVmwareDc", description = "Adds a Vmware datacenter to specified zone", + responseObject = VmwareDatacenterResponse.class, responseHasSensitiveInfo = false) public class AddVmwareDcCmd extends BaseCmd { @Inject @@ -45,7 +45,7 @@ public class AddVmwareDcCmd extends BaseCmd { - @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, required = true, description = "Name of VMware datacenter to be added to specified zone.") + @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, required = true, description = "Name of Vmware datacenter to be added to specified zone.") private String name; @Parameter(name = ApiConstants.VCENTER, @@ -54,10 +54,10 @@ public class AddVmwareDcCmd extends BaseCmd { description = "The name/ip of vCenter. Make sure it is IP address or full qualified domain name for host running vCenter server.") private String vCenter; - @Parameter(name = ApiConstants.USERNAME, type = CommandType.STRING, required = false, description = "The Username required to connect to resource.") + @Parameter(name = ApiConstants.USERNAME, type = CommandType.STRING, description = "The Username required to connect to resource.") private String username; - @Parameter(name = ApiConstants.PASSWORD, type = CommandType.STRING, required = false, description = "The password for specified username.") + @Parameter(name = ApiConstants.PASSWORD, type = CommandType.STRING, description = "The password for specified username.") private String password; @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.UUID, entityType = ZoneResponse.class, required = true, description = "The Zone ID.") @@ -99,7 +99,7 @@ public class AddVmwareDcCmd extends BaseCmd { response.setResponseName(getCommandName()); response.setObjectName("vmwaredc"); } else { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to add VMware Datacenter to zone."); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to add Vmware Datacenter to zone."); } this.setResponseObject(response); } catch (DiscoveryException ex) { diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ImportVsphereStoragePoliciesCmd.java b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ImportVsphereStoragePoliciesCmd.java index 0d8d5d6fa07..40b479f809c 100644 --- a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ImportVsphereStoragePoliciesCmd.java +++ b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ImportVsphereStoragePoliciesCmd.java @@ -48,8 +48,6 @@ import java.util.List; authorized = {RoleType.Admin}) public class ImportVsphereStoragePoliciesCmd extends BaseCmd { - - @Inject public VmwareDatacenterService _vmwareDatacenterService; @@ -74,6 +72,13 @@ public class ImportVsphereStoragePoliciesCmd extends BaseCmd { List storagePolicies = _vmwareDatacenterService.importVsphereStoragePolicies(this); final ListResponse responseList = new ListResponse<>(); + final List storagePoliciesResponseList = getVsphereStoragePoliciesResponses(storagePolicies, dataCenter); + responseList.setResponses(storagePoliciesResponseList); + responseList.setResponseName(getCommandName()); + setResponseObject(responseList); + } + + private static List getVsphereStoragePoliciesResponses(List storagePolicies, DataCenter dataCenter) { final List storagePoliciesResponseList = new ArrayList<>(); for (VsphereStoragePolicy storagePolicy : storagePolicies) { final VsphereStoragePoliciesResponse storagePoliciesResponse = new VsphereStoragePoliciesResponse(); @@ -86,9 +91,7 @@ public class ImportVsphereStoragePoliciesCmd extends BaseCmd { storagePoliciesResponseList.add(storagePoliciesResponse); } - responseList.setResponses(storagePoliciesResponseList); - responseList.setResponseName(getCommandName()); - setResponseObject(responseList); + return storagePoliciesResponseList; } @Override diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVmwareDcHostsCmd.java b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVmwareDcHostsCmd.java new file mode 100644 index 00000000000..6f193c9c1b2 --- /dev/null +++ b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVmwareDcHostsCmd.java @@ -0,0 +1,144 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.zone; + +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.hypervisor.vmware.VmwareDatacenterService; +import com.cloud.hypervisor.vmware.mo.HostMO; +import com.cloud.user.Account; +import com.cloud.utils.exception.CloudRuntimeException; + +import com.vmware.vim25.InvalidPropertyFaultMsg; +import com.vmware.vim25.RuntimeFaultFaultMsg; + +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.BaseResponse; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.HostResponse; +import org.apache.cloudstack.api.response.VmwareDatacenterResponse; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; + +import javax.inject.Inject; +import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.List; + +@APICommand(name = "listVmwareDcHosts", responseObject = VmwareRequestResponse.class, + description = "Lists the VMs in a Vmware Datacenter", + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) +public class ListVmwareDcHostsCmd extends BaseCmd implements ListVmwareDcItems { + + @Inject + public VmwareDatacenterService _vmwareDatacenterService; + + @Parameter(name = ApiConstants.EXISTING_VCENTER_ID, + type = CommandType.UUID, + entityType = VmwareDatacenterResponse.class, + description = "UUID of a linked existing vCenter") + private Long existingVcenterId; + + @Parameter(name = ApiConstants.VCENTER, + type = CommandType.STRING, + description = "The name/ip of vCenter. Make sure it is IP address or full qualified domain name for host running vCenter server.") + private String vcenter; + + @Parameter(name = ApiConstants.DATACENTER_NAME, type = CommandType.STRING, description = "Name of Vmware datacenter.") + private String datacenterName; + + @Parameter(name = ApiConstants.USERNAME, type = CommandType.STRING, description = "The Username required to connect to resource.") + private String username; + + @Parameter(name = ApiConstants.PASSWORD, type = CommandType.STRING, description = "The password for specified username.") + private String password; + + public String getVcenter() { + return vcenter; + } + + public String getUsername() { + return username; + } + + public String getPassword() { + return password; + } + + public String getDatacenterName() { + return datacenterName; + } + + public Long getExistingVcenterId() { + return existingVcenterId; + } + + @Override + public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { + checkParameters(); + try { + List hosts = _vmwareDatacenterService.listHostsInDatacenter(this); + List baseResponseList = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(hosts)) { + for (HostMO vmwareHost : hosts) { + HostResponse resp = createHostResponse(vmwareHost); + baseResponseList.add(resp); + } + } + VmwareRequestResponse response = new VmwareRequestResponse<>(); + response.setResponses(baseResponseList, baseResponseList.size()); + response.setResponseName(getCommandName()); + setResponseObject(response); + } catch (CloudRuntimeException | InvalidPropertyFaultMsg | RuntimeFaultFaultMsg | InvocationTargetException | + NoSuchMethodException | IllegalAccessException e) { + String errorMsg = String.format("Error retrieving VMs from Vmware VC: %s", e.getMessage()); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, errorMsg); + } + } + + private HostResponse createHostResponse(HostMO hostInstance) throws InvalidPropertyFaultMsg, RuntimeFaultFaultMsg, InvocationTargetException, NoSuchMethodException, IllegalAccessException { + HostResponse response = new HostResponse(); + response.setHypervisor(Hypervisor.HypervisorType.VMware.toString()); + response.setName(hostInstance.getHostName()); + response.setObjectName("host"); + return response; + } + + private void checkParameters() { + if ((existingVcenterId == null && vcenter == null) || (existingVcenterId != null && vcenter != null)) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, + "Please provide an existing vCenter ID or a vCenter IP/Name, parameters are mutually exclusive"); + } + if (existingVcenterId == null && StringUtils.isAnyBlank(vcenter, datacenterName, username, password)) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, + "Please set all the information for a vCenter IP/Name, datacenter, username and password"); + } + } + + @Override + public long getEntityOwnerId() { + return Account.ACCOUNT_ID_SYSTEM; + } +} diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVmwareDcItems.java b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVmwareDcItems.java new file mode 100644 index 00000000000..580fb3bad9b --- /dev/null +++ b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVmwareDcItems.java @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.zone; + +public interface ListVmwareDcItems { + String getVcenter(); + + String getDatacenterName(); + + String getUsername(); + + String getPassword(); + + Long getExistingVcenterId(); +} diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVmwareDcVmsCmd.java b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVmwareDcVmsCmd.java index 4dd1b4beb09..544e756fe80 100644 --- a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVmwareDcVmsCmd.java +++ b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVmwareDcVmsCmd.java @@ -23,15 +23,15 @@ import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; import com.cloud.hypervisor.vmware.VmwareDatacenterService; import com.cloud.user.Account; +import com.cloud.utils.Pair; import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; -import org.apache.cloudstack.api.BaseListCmd; +import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.BaseResponse; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ServerApiException; -import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.UnmanagedInstanceResponse; import org.apache.cloudstack.api.response.VmwareDatacenterResponse; import org.apache.cloudstack.vm.UnmanagedInstanceTO; @@ -42,10 +42,10 @@ import javax.inject.Inject; import java.util.ArrayList; import java.util.List; -@APICommand(name = "listVmwareDcVms", responseObject = UnmanagedInstanceResponse.class, - description = "Lists the VMs in a VMware Datacenter", +@APICommand(name = "listVmwareDcVms", responseObject = VmwareRequestResponse.class, + description = "Lists the VMs in a Vmware Datacenter", requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) -public class ListVmwareDcVmsCmd extends BaseListCmd { +public class ListVmwareDcVmsCmd extends BaseCmd implements ListVmwareDcItems { @Inject public VmwareDatacenterService _vmwareDatacenterService; @@ -61,7 +61,7 @@ public class ListVmwareDcVmsCmd extends BaseListCmd { description = "The name/ip of vCenter. Make sure it is IP address or full qualified domain name for host running vCenter server.") private String vcenter; - @Parameter(name = ApiConstants.DATACENTER_NAME, type = CommandType.STRING, description = "Name of VMware datacenter.") + @Parameter(name = ApiConstants.DATACENTER_NAME, type = CommandType.STRING, description = "Name of Vmware datacenter.") private String datacenterName; @Parameter(name = ApiConstants.USERNAME, type = CommandType.STRING, description = "The Username required to connect to resource.") @@ -70,6 +70,18 @@ public class ListVmwareDcVmsCmd extends BaseListCmd { @Parameter(name = ApiConstants.PASSWORD, type = CommandType.STRING, description = "The password for specified username.") private String password; + @Parameter(name = ApiConstants.HOST, type = CommandType.STRING, description = "get only the VMs from the specified host.") + private String host; + + @Parameter(name = ApiConstants.BATCH_SIZE, type = CommandType.INTEGER, description = "The maximum number of results to return.") + private Integer batchSize; + + @Parameter(name = ApiConstants.TOKEN, type = CommandType.STRING, + description = "For listVmwareDcVms, if the maximum number of results (the `batchsize`) is exceeded, " + + " a token is returned. This token can be used in subsequent calls to retrieve more results." + + " As long as a token is returned, more results can be retrieved.") + private String token; + public String getVcenter() { return vcenter; } @@ -82,6 +94,18 @@ public class ListVmwareDcVmsCmd extends BaseListCmd { return password; } + public Integer getBatchSize() { + return batchSize; + } + + public String getHost() { + return host; + } + + public String getToken() { + return token; + } + public String getDatacenterName() { return datacenterName; } @@ -94,7 +118,8 @@ public class ListVmwareDcVmsCmd extends BaseListCmd { public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { checkParameters(); try { - List vms = _vmwareDatacenterService.listVMsInDatacenter(this); + Pair> results = _vmwareDatacenterService.listVMsInDatacenter(this); + List vms = results.second(); List baseResponseList = new ArrayList<>(); if (CollectionUtils.isNotEmpty(vms)) { for (UnmanagedInstanceTO vmwareVm : vms) { @@ -102,16 +127,13 @@ public class ListVmwareDcVmsCmd extends BaseListCmd { baseResponseList.add(resp); } } - List pagingList = com.cloud.utils.StringUtils.applyPagination(baseResponseList, this.getStartIndex(), this.getPageSizeVal()); - if (CollectionUtils.isEmpty(pagingList)) { - pagingList = baseResponseList; - } - ListResponse response = new ListResponse<>(); - response.setResponses(pagingList, baseResponseList.size()); + VmwareRequestResponse response = new VmwareRequestResponse<>(); + response.setResponses(baseResponseList, baseResponseList.size()); response.setResponseName(getCommandName()); + response.setToken(results.first()); setResponseObject(response); } catch (CloudRuntimeException e) { - String errorMsg = String.format("Error retrieving VMs from VMware VC: %s", e.getMessage()); + String errorMsg = String.format("Error retrieving VMs from Vmware VC: %s", e.getMessage()); throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, errorMsg); } } @@ -134,6 +156,6 @@ public class ListVmwareDcVmsCmd extends BaseListCmd { @Override public String getCommandName() { - return "listvmwaredcvmsresponse"; + return "listVmwareDcVmsResponse".toLowerCase(); } } diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVmwareDcsCmd.java b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVmwareDcsCmd.java index 4c7f2a5c7d8..9cafda955ca 100644 --- a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVmwareDcsCmd.java +++ b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVmwareDcsCmd.java @@ -42,7 +42,7 @@ import com.cloud.dc.VmwareDatacenter; import com.cloud.hypervisor.vmware.VmwareDatacenterService; import com.cloud.user.Account; -@APICommand(name = "listVmwareDcs", responseObject = VmwareDatacenterResponse.class, description = "Retrieves VMware DC(s) associated with a zone.", +@APICommand(name = "listVmwareDcs", responseObject = VmwareDatacenterResponse.class, description = "Retrieves Vmware DC(s) associated with a zone.", requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) public class ListVmwareDcsCmd extends BaseListCmd { @@ -50,7 +50,6 @@ public class ListVmwareDcsCmd extends BaseListCmd { public VmwareDatacenterService _vmwareDatacenterService; - ///////////////////////////////////////////////////// //////////////// API parameters ///////////////////// ///////////////////////////////////////////////////// @@ -73,20 +72,27 @@ public class ListVmwareDcsCmd extends BaseListCmd { @Override public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException { - List vmwareDcList = null; + List vmwareDcList; try { vmwareDcList = _vmwareDatacenterService.listVmwareDatacenters(this); } catch (InvalidParameterValueException ie) { throw new InvalidParameterValueException("Invalid zone id " + getZoneId()); } catch (Exception e) { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to find associated VMware DCs associated with zone " + getZoneId()); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to find associated Vmware DCs associated with zone " + getZoneId()); } - ListResponse response = new ListResponse(); - List vmwareDcResponses = new ArrayList(); + ListResponse response = new ListResponse<>(); + List vmwareDcResponses = getVmwareDatacenterResponses(vmwareDcList); + response.setResponses(vmwareDcResponses); + response.setResponseName(getCommandName()); + setResponseObject(response); + } - if (vmwareDcList != null && vmwareDcList.size() > 0) { + private List getVmwareDatacenterResponses(List vmwareDcList) { + List vmwareDcResponses = new ArrayList<>(); + + if (vmwareDcList != null && !vmwareDcList.isEmpty()) { for (VmwareDatacenter vmwareDc : vmwareDcList) { VmwareDatacenterResponse vmwareDcResponse = new VmwareDatacenterResponse(); @@ -94,14 +100,12 @@ public class ListVmwareDcsCmd extends BaseListCmd { vmwareDcResponse.setVcenter(vmwareDc.getVcenterHost()); vmwareDcResponse.setName(vmwareDc.getVmwareDatacenterName()); vmwareDcResponse.setZoneId(getZoneId()); - vmwareDcResponse.setObjectName("VMwareDC"); + vmwareDcResponse.setObjectName(ApiConstants.VMWARE_DC); vmwareDcResponses.add(vmwareDcResponse); } } - response.setResponses(vmwareDcResponses); - response.setResponseName(getCommandName()); - setResponseObject(response); + return vmwareDcResponses; } @Override diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVsphereStoragePoliciesCmd.java b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVsphereStoragePoliciesCmd.java index c8527b1ec34..35631ba1315 100644 --- a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVsphereStoragePoliciesCmd.java +++ b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVsphereStoragePoliciesCmd.java @@ -47,8 +47,6 @@ import java.util.List; authorized = {RoleType.Admin}) public class ListVsphereStoragePoliciesCmd extends BaseCmd { - - @Inject public VmwareDatacenterService _vmwareDatacenterService; @@ -73,6 +71,13 @@ public class ListVsphereStoragePoliciesCmd extends BaseCmd { List storagePolicies = _vmwareDatacenterService.listVsphereStoragePolicies(this); final ListResponse responseList = new ListResponse<>(); + final List storagePoliciesResponseList = getVsphereStoragePoliciesResponses(storagePolicies, dataCenter); + responseList.setResponses(storagePoliciesResponseList); + responseList.setResponseName(getCommandName()); + setResponseObject(responseList); + } + + private static List getVsphereStoragePoliciesResponses(List storagePolicies, DataCenter dataCenter) { final List storagePoliciesResponseList = new ArrayList<>(); for (VsphereStoragePolicy storagePolicy : storagePolicies) { final VsphereStoragePoliciesResponse storagePoliciesResponse = new VsphereStoragePoliciesResponse(); @@ -81,13 +86,11 @@ public class ListVsphereStoragePoliciesCmd extends BaseCmd { storagePoliciesResponse.setName(storagePolicy.getName()); storagePoliciesResponse.setPolicyId(storagePolicy.getPolicyId()); storagePoliciesResponse.setDescription(storagePolicy.getDescription()); - storagePoliciesResponse.setObjectName("StoragePolicy"); + storagePoliciesResponse.setObjectName(ApiConstants.STORAGE_POLICY); storagePoliciesResponseList.add(storagePoliciesResponse); } - responseList.setResponses(storagePoliciesResponseList); - responseList.setResponseName(getCommandName()); - setResponseObject(responseList); + return storagePoliciesResponseList; } @Override diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVsphereStoragePolicyCompatiblePoolsCmd.java b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVsphereStoragePolicyCompatiblePoolsCmd.java index d66972ded2e..ab697fbad67 100644 --- a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVsphereStoragePolicyCompatiblePoolsCmd.java +++ b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/ListVsphereStoragePolicyCompatiblePoolsCmd.java @@ -68,7 +68,7 @@ public class ListVsphereStoragePolicyCompatiblePoolsCmd extends BaseListCmd { List poolResponses = new ArrayList<>(); for (StoragePool pool : pools) { StoragePoolResponse poolResponse = _responseGenerator.createStoragePoolForMigrationResponse(pool); - poolResponse.setObjectName("storagepool"); + poolResponse.setObjectName(ApiConstants.STORAGE_POOL); poolResponses.add(poolResponse); } response.setResponses(poolResponses); diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/RemoveVmwareDcCmd.java b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/RemoveVmwareDcCmd.java index a503d860feb..d85fbacf2db 100644 --- a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/RemoveVmwareDcCmd.java +++ b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/RemoveVmwareDcCmd.java @@ -34,7 +34,7 @@ import com.cloud.hypervisor.vmware.VmwareDatacenterService; import com.cloud.user.Account; import com.cloud.utils.exception.CloudRuntimeException; -@APICommand(name = "removeVmwareDc", responseObject = SuccessResponse.class, description = "Remove a VMware datacenter from a zone.", +@APICommand(name = "removeVmwareDc", responseObject = SuccessResponse.class, description = "Remove a Vmware datacenter from a zone.", requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) public class RemoveVmwareDcCmd extends BaseCmd { @@ -47,7 +47,7 @@ public class RemoveVmwareDcCmd extends BaseCmd { type = CommandType.UUID, entityType = ZoneResponse.class, required = true, - description = "The id of Zone from which VMware datacenter has to be removed.") + description = "The id of Zone from which Vmware datacenter has to be removed.") private Long zoneId; public Long getZoneId() { @@ -63,7 +63,7 @@ public class RemoveVmwareDcCmd extends BaseCmd { response.setResponseName(getCommandName()); setResponseObject(response); } else { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to remove VMware datacenter from zone"); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to remove Vmware datacenter from zone"); } } catch (ResourceInUseException ex) { logger.warn("The zone has one or more resources (like cluster), hence not able to remove VMware datacenter from zone." diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/UpdateVmwareDcCmd.java b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/UpdateVmwareDcCmd.java index bb818985277..5e02d5a96c2 100644 --- a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/UpdateVmwareDcCmd.java +++ b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/UpdateVmwareDcCmd.java @@ -33,12 +33,11 @@ import com.cloud.dc.VmwareDatacenter; import com.cloud.hypervisor.vmware.VmwareDatacenterService; import com.cloud.user.Account; -@APICommand(name = "updateVmwareDc", description = "Updates a VMware datacenter details for a zone", +@APICommand(name = "updateVmwareDc", description = "Updates a Vmware datacenter details for a zone", responseObject = VmwareDatacenterResponse.class, responseHasSensitiveInfo = false, since = "4.12.0", authorized = {RoleType.Admin}) public class UpdateVmwareDcCmd extends BaseCmd { - @Inject public VmwareDatacenterService vmwareDatacenterService; @@ -51,7 +50,7 @@ public class UpdateVmwareDcCmd extends BaseCmd { private Long zoneId; @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, - description = "VMware datacenter name.") + description = "Vmware datacenter name.") private String name; @Parameter(name = ApiConstants.VCENTER, type = CommandType.STRING, @@ -106,13 +105,13 @@ public class UpdateVmwareDcCmd extends BaseCmd { public void execute() { final VmwareDatacenter vmwareDatacenter = vmwareDatacenterService.updateVmwareDatacenter(this); if (vmwareDatacenter == null) { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to update VMware datacenter"); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to update Vmware datacenter"); } final VmwareDatacenterResponse response = new VmwareDatacenterResponse(); response.setId(vmwareDatacenter.getUuid()); response.setName(vmwareDatacenter.getVmwareDatacenterName()); response.setResponseName(getCommandName()); - response.setObjectName("vmwaredc"); + response.setObjectName(ApiConstants.VMWARE_DC); setResponseObject(response); } diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/VmwareRequestResponse.java b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/VmwareRequestResponse.java new file mode 100644 index 00000000000..81c58ef27ba --- /dev/null +++ b/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/VmwareRequestResponse.java @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.admin.zone; + +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.response.ListResponse; + +public class VmwareRequestResponse extends ListResponse { + @SerializedName(ApiConstants.TOKEN) + @Param(description = "The Vmware API token to use for retrieving further responses with") + private String token; + + public String getToken() { + return token; + } + + public void setToken(String token) { + this.token = token; + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index 669c0a1a0a0..ccc80daa7d7 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -110,6 +110,7 @@ import org.apache.cloudstack.network.RoutedIpv4Manager; import org.apache.commons.codec.binary.Base64; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import com.cloud.api.ApiDBUtils; @@ -454,10 +455,10 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne } protected void validateIsolatedNetworkIpRules(long ipId, FirewallRule.Purpose purpose, Network network, int clusterTotalNodeCount) { - List rules = firewallRulesDao.listByIpAndPurposeAndNotRevoked(ipId, purpose); + List rules = firewallRulesDao.listByIpPurposeProtocolAndNotRevoked(ipId, purpose, NetUtils.TCP_PROTO); for (FirewallRuleVO rule : rules) { - Integer startPort = rule.getSourcePortStart(); - Integer endPort = rule.getSourcePortEnd(); + int startPort = ObjectUtils.defaultIfNull(rule.getSourcePortStart(), 1); + int endPort = ObjectUtils.defaultIfNull(rule.getSourcePortEnd(), NetUtils.PORT_RANGE_MAX); if (logger.isDebugEnabled()) { logger.debug("Validating rule with purpose: {} for network: {} with ports: {}-{}", purpose.toString(), network, startPort, endPort); } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java index eba2b8535ab..bc5af08a9c1 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java @@ -25,11 +25,9 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; @@ -268,14 +266,12 @@ public class KubernetesClusterActionWorker { protected String getControlNodeLoginUser() { List vmMapVOList = getKubernetesClusterVMMaps(); - if (vmMapVOList.size() > 0) { + if (!vmMapVOList.isEmpty()) { long vmId = vmMapVOList.get(0).getVmId(); UserVmVO userVM = userVmDao.findById(vmId); if (userVM == null) { throw new CloudRuntimeException("Failed to find login user, Unable to log in to node to fetch details"); } - Set vm = new HashSet<>(); - vm.add(userVM.getName()); UserVmDetailVO vmDetail = userVmDetailsDao.findDetail(vmId, VmDetailConstants.CKS_CONTROL_NODE_LOGIN_USER); if (vmDetail != null && !org.apache.commons.lang3.StringUtils.isEmpty(vmDetail.getValue())) { return vmDetail.getValue(); @@ -434,7 +430,12 @@ public class KubernetesClusterActionWorker { } IpAddress address = ipAddressDao.findByUuid(detailsVO.getValue()); if (address == null || !Objects.equals(network.getVpcId(), address.getVpcId())) { - logger.warn(String.format("Public IP with ID: %s linked to the Kubernetes cluster: %s is not usable", detailsVO.getValue(), kubernetesCluster.getName())); + logger.warn("Public IP with ID: {} linked to the Kubernetes cluster: {} is not usable", detailsVO.getValue(), kubernetesCluster.getName()); + if (address == null) { + logger.warn("Public IP with ID: {} was not found by uuid", detailsVO.getValue()); + } else { + logger.warn("Public IP with ID: {} was associated with vpc {} instead of {}", detailsVO.getValue(), address.getVpcId(), network.getVpcId()); + } return null; } return address; @@ -618,8 +619,7 @@ public class KubernetesClusterActionWorker { } protected List getKubernetesClusterVMMaps() { - List clusterVMs = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); - return clusterVMs; + return kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); } protected List getKubernetesClusterVMMapsForNodes(List nodeIds) { diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java index b84559797f6..7d96e022a8c 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java @@ -516,10 +516,12 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu protected FirewallRule removeApiFirewallRule(final IpAddress publicIp) { FirewallRule rule = null; - List firewallRules = firewallRulesDao.listByIpAndPurposeAndNotRevoked(publicIp.getId(), FirewallRule.Purpose.Firewall); + List firewallRules = firewallRulesDao.listByIpPurposeProtocolAndNotRevoked(publicIp.getId(), FirewallRule.Purpose.Firewall, NetUtils.TCP_PROTO); for (FirewallRuleVO firewallRule : firewallRules) { - if (firewallRule.getSourcePortStart() == CLUSTER_API_PORT && - firewallRule.getSourcePortEnd() == CLUSTER_API_PORT) { + Integer startPort = firewallRule.getSourcePortStart(); + Integer endPort = firewallRule.getSourcePortEnd(); + if (startPort != null && startPort == CLUSTER_API_PORT && + endPort != null && endPort == CLUSTER_API_PORT) { rule = firewallRule; firewallService.revokeIngressFwRule(firewallRule.getId(), true); logger.debug("The API firewall rule [%s] with the id [%s] was revoked",firewallRule.getName(),firewallRule.getId()); @@ -531,7 +533,7 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu protected FirewallRule removeSshFirewallRule(final IpAddress publicIp, final long networkId) { FirewallRule rule = null; - List firewallRules = firewallRulesDao.listByIpAndPurposeAndNotRevoked(publicIp.getId(), FirewallRule.Purpose.Firewall); + List firewallRules = firewallRulesDao.listByIpPurposeProtocolAndNotRevoked(publicIp.getId(), FirewallRule.Purpose.Firewall, NetUtils.TCP_PROTO); for (FirewallRuleVO firewallRule : firewallRules) { PortForwardingRuleVO pfRule = portForwardingRulesDao.findByNetworkAndPorts(networkId, firewallRule.getSourcePortStart(), firewallRule.getSourcePortEnd()); if (firewallRule.getSourcePortStart() == CLUSTER_NODES_DEFAULT_START_SSH_PORT || (Objects.nonNull(pfRule) && pfRule.getDestinationPortStart() == DEFAULT_SSH_PORT) ) { diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java index 2c991c94458..00625f6e076 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java @@ -195,7 +195,7 @@ public class KubernetesClusterUtil { while (System.currentTimeMillis() < timeoutTime) { try { Pair result = SshHelper.sshExecute(ipAddress, port, user, - sshKeyFile, null, "sudo cat /etc/kubernetes/admin.conf", + sshKeyFile, null, "sudo cat /etc/kubernetes/user.conf 2>/dev/null || sudo cat /etc/kubernetes/admin.conf", 10000, 10000, 10000); if (result.first() && StringUtils.isNotEmpty(result.second())) { diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java index a9cb7096b8a..57715cc9309 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java @@ -42,6 +42,7 @@ import com.cloud.user.Account; import com.cloud.user.AccountManager; import com.cloud.user.User; import com.cloud.utils.Pair; +import com.cloud.utils.net.NetUtils; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.dao.VMInstanceDao; import org.apache.cloudstack.api.BaseCmd; @@ -133,7 +134,7 @@ public class KubernetesClusterManagerImplTest { long ipId = 1L; FirewallRule.Purpose purpose = FirewallRule.Purpose.Firewall; Network network = Mockito.mock(Network.class); - Mockito.when(firewallRulesDao.listByIpAndPurposeAndNotRevoked(ipId, purpose)).thenReturn(new ArrayList<>()); + Mockito.when(firewallRulesDao.listByIpPurposeProtocolAndNotRevoked(ipId, purpose, NetUtils.TCP_PROTO)).thenReturn(new ArrayList<>()); kubernetesClusterManager.validateIsolatedNetworkIpRules(ipId, FirewallRule.Purpose.Firewall, network, 3); } @@ -147,7 +148,7 @@ public class KubernetesClusterManagerImplTest { long ipId = 1L; FirewallRule.Purpose purpose = FirewallRule.Purpose.Firewall; Network network = Mockito.mock(Network.class); - Mockito.when(firewallRulesDao.listByIpAndPurposeAndNotRevoked(ipId, purpose)).thenReturn(List.of(createRule(80, 80), createRule(443, 443))); + Mockito.when(firewallRulesDao.listByIpPurposeProtocolAndNotRevoked(ipId, purpose, NetUtils.TCP_PROTO)).thenReturn(List.of(createRule(80, 80), createRule(443, 443))); kubernetesClusterManager.validateIsolatedNetworkIpRules(ipId, FirewallRule.Purpose.Firewall, network, 3); } @@ -156,7 +157,7 @@ public class KubernetesClusterManagerImplTest { long ipId = 1L; FirewallRule.Purpose purpose = FirewallRule.Purpose.Firewall; Network network = Mockito.mock(Network.class); - Mockito.when(firewallRulesDao.listByIpAndPurposeAndNotRevoked(ipId, purpose)).thenReturn(List.of(createRule(6440, 6445), createRule(443, 443))); + Mockito.when(firewallRulesDao.listByIpPurposeProtocolAndNotRevoked(ipId, purpose, NetUtils.TCP_PROTO)).thenReturn(List.of(createRule(6440, 6445), createRule(443, 443))); kubernetesClusterManager.validateIsolatedNetworkIpRules(ipId, FirewallRule.Purpose.Firewall, network, 3); } @@ -165,7 +166,7 @@ public class KubernetesClusterManagerImplTest { long ipId = 1L; FirewallRule.Purpose purpose = FirewallRule.Purpose.Firewall; Network network = Mockito.mock(Network.class); - Mockito.when(firewallRulesDao.listByIpAndPurposeAndNotRevoked(ipId, purpose)).thenReturn(List.of(createRule(2200, KubernetesClusterActionWorker.CLUSTER_NODES_DEFAULT_START_SSH_PORT), createRule(443, 443))); + Mockito.when(firewallRulesDao.listByIpPurposeProtocolAndNotRevoked(ipId, purpose, NetUtils.TCP_PROTO)).thenReturn(List.of(createRule(2200, KubernetesClusterActionWorker.CLUSTER_NODES_DEFAULT_START_SSH_PORT), createRule(443, 443))); kubernetesClusterManager.validateIsolatedNetworkIpRules(ipId, FirewallRule.Purpose.Firewall, network, 3); } @@ -174,7 +175,7 @@ public class KubernetesClusterManagerImplTest { long ipId = 1L; FirewallRule.Purpose purpose = FirewallRule.Purpose.Firewall; Network network = Mockito.mock(Network.class); - Mockito.when(firewallRulesDao.listByIpAndPurposeAndNotRevoked(ipId, purpose)).thenReturn(List.of(createRule(2220, 2221), createRule(2225, 2227), createRule(6440, 6442), createRule(6444, 6446))); + Mockito.when(firewallRulesDao.listByIpPurposeProtocolAndNotRevoked(ipId, purpose, NetUtils.TCP_PROTO)).thenReturn(List.of(createRule(2220, 2221), createRule(2225, 2227), createRule(6440, 6442), createRule(6444, 6446))); kubernetesClusterManager.validateIsolatedNetworkIpRules(ipId, FirewallRule.Purpose.Firewall, network, 3); } diff --git a/plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsServiceImpl.java b/plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsServiceImpl.java index bd13747bd75..c4bf3e60d82 100644 --- a/plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsServiceImpl.java +++ b/plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsServiceImpl.java @@ -80,6 +80,7 @@ import org.apache.cloudstack.response.ZoneMetricsResponse; import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.utils.bytescale.ByteScaleUtils; import org.apache.commons.beanutils.BeanUtils; import org.apache.commons.collections.CollectionUtils; @@ -627,6 +628,7 @@ public class MetricsServiceImpl extends MutualExclusiveIdsManagerBase implements public List listStoragePoolMetrics(List poolResponses) { final List metricsResponses = new ArrayList<>(); Map clusterUuidToIdMap = clusterDao.findByUuids(poolResponses.stream().map(StoragePoolResponse::getClusterId).toArray(String[]::new)).stream().collect(Collectors.toMap(ClusterVO::getUuid, ClusterVO::getId)); + Map poolUuidToIdMap = storagePoolDao.findByUuids(poolResponses.stream().map(StoragePoolResponse::getId).toArray(String[]::new)).stream().collect(Collectors.toMap(StoragePoolVO::getUuid, StoragePoolVO::getId)); for (final StoragePoolResponse poolResponse: poolResponses) { StoragePoolMetricsResponse metricsResponse = new StoragePoolMetricsResponse(); @@ -637,8 +639,9 @@ public class MetricsServiceImpl extends MutualExclusiveIdsManagerBase implements } Long poolClusterId = clusterUuidToIdMap.get(poolResponse.getClusterId()); + Long poolId = poolUuidToIdMap.get(poolResponse.getId()); final Double storageThreshold = AlertManager.StorageCapacityThreshold.valueIn(poolClusterId); - final Double storageDisableThreshold = CapacityManager.StorageCapacityDisableThreshold.valueIn(poolClusterId); + final Double storageDisableThreshold = CapacityManager.StorageCapacityDisableThreshold.valueIn(poolId); metricsResponse.setHasAnnotation(poolResponse.hasAnnotation()); metricsResponse.setDiskSizeUsedGB(poolResponse.getDiskSizeUsed()); diff --git a/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java b/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java index 551d96eab9a..7eb350a0643 100644 --- a/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java +++ b/plugins/storage/object/ceph/src/main/java/org/apache/cloudstack/storage/datastore/driver/CephObjectStoreDriverImpl.java @@ -290,7 +290,7 @@ public class CephObjectStoreDriverImpl extends BaseObjectStoreDriverImpl { RgwAdmin rgwAdmin = getRgwAdminClient(storeId); try { - rgwAdmin.setBucketQuota(bucket.getName(), -1, size); + rgwAdmin.setIndividualBucketQuota(null, bucket.getName(), -1, size * 1024 * 1024); } catch (Exception e) { throw new CloudRuntimeException(e); } diff --git a/plugins/storage/volume/linstor/CHANGELOG.md b/plugins/storage/volume/linstor/CHANGELOG.md index 419a7f983ee..fb247eef5df 100644 --- a/plugins/storage/volume/linstor/CHANGELOG.md +++ b/plugins/storage/volume/linstor/CHANGELOG.md @@ -5,12 +5,23 @@ All notable changes to Linstor CloudStack plugin will be documented in this file The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [2025-01-27] + +### Fixed + +- Use of multiple primary storages on the same linstor controller + ## [2025-01-20] ### Fixed - Volume snapshots on zfs used the wrong dataset path to hide/unhide snapdev +## [2024-12-19] + +### Added +- Native CloudStack encryption support + ## [2024-12-13] ### Fixed diff --git a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LinstorBackupSnapshotCommandWrapper.java b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LinstorBackupSnapshotCommandWrapper.java index 08f61262914..fab4829da55 100644 --- a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LinstorBackupSnapshotCommandWrapper.java +++ b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LinstorBackupSnapshotCommandWrapper.java @@ -97,18 +97,23 @@ public final class LinstorBackupSnapshotCommandWrapper // NOTE: the qemu img will also contain the drbd metadata at the end final QemuImg qemu = new QemuImg(waitMilliSeconds); qemu.convert(srcFile, dstFile); - LOGGER.info("Backup snapshot " + srcFile + " to " + dstPath); + LOGGER.info("Backup snapshot '{}' to '{}'", srcPath, dstPath); return dstPath; } private SnapshotObjectTO setCorrectSnapshotSize(final SnapshotObjectTO dst, final String dstPath) { final File snapFile = new File(dstPath); - final long size = snapFile.exists() ? snapFile.length() : 0; + long size; + if (snapFile.exists()) { + size = snapFile.length(); + } else { + LOGGER.warn("Snapshot file {} does not exist. Reporting size 0", dstPath); + size = 0; + } - final SnapshotObjectTO snapshot = new SnapshotObjectTO(); - snapshot.setPath(dst.getPath() + File.separator + dst.getName()); - snapshot.setPhysicalSize(size); - return snapshot; + dst.setPath(dst.getPath() + File.separator + dst.getName()); + dst.setPhysicalSize(size); + return dst; } @Override @@ -158,6 +163,7 @@ public final class LinstorBackupSnapshotCommandWrapper LOGGER.info("Backup shrunk " + dstPath + " to actual size " + src.getVolume().getSize()); SnapshotObjectTO snapshot = setCorrectSnapshotSize(dst, dstPath); + LOGGER.info("Actual file size for '{}' is {}", dstPath, snapshot.getPhysicalSize()); return new CopyCmdAnswer(snapshot); } catch (final Exception e) { final String error = String.format("Failed to backup snapshot with id [%s] with a pool %s, due to %s", diff --git a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java index 292e939de49..ba4a7b14787 100644 --- a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java +++ b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java @@ -61,6 +61,11 @@ import com.linbit.linstor.api.model.Volume; import com.linbit.linstor.api.model.VolumeDefinition; import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; public class LinstorStorageAdaptor implements StorageAdaptor { protected Logger logger = LogManager.getLogger(getClass()); @@ -202,10 +207,10 @@ public class LinstorStorageAdaptor implements StorageAdaptor { final DevelopersApi api = getLinstorAPI(pool); try { - List definitionList = api.resourceDefinitionList( - Collections.singletonList(rscName), null, null, null); + ResourceDefinition resourceDefinition = LinstorUtil.findResourceDefinition( + api, rscName, lpool.getResourceGroup()); - if (definitionList.isEmpty()) { + if (resourceDefinition == null) { ResourceGroupSpawn rgSpawn = new ResourceGroupSpawn(); rgSpawn.setResourceDefinitionName(rscName); rgSpawn.addVolumeSizesItem(size / 1024); // linstor uses KiB @@ -215,22 +220,28 @@ public class LinstorStorageAdaptor implements StorageAdaptor { handleLinstorApiAnswers(answers, "Linstor: Unable to spawn resource."); } + String foundRscName = resourceDefinition != null ? resourceDefinition.getName() : rscName; + // query linstor for the device path List resources = api.viewResources( Collections.emptyList(), - Collections.singletonList(rscName), + Collections.singletonList(foundRscName), Collections.emptyList(), null, null, null); - makeResourceAvailable(api, rscName, false); + makeResourceAvailable(api, foundRscName, false); if (!resources.isEmpty() && !resources.get(0).getVolumes().isEmpty()) { final String devPath = resources.get(0).getVolumes().get(0).getDevicePath(); logger.info("Linstor: Created drbd device: " + devPath); final KVMPhysicalDisk kvmDisk = new KVMPhysicalDisk(devPath, name, pool); kvmDisk.setFormat(QemuImg.PhysicalDiskFormat.RAW); + long allocatedKib = resources.get(0).getVolumes().get(0).getAllocatedSizeKib() != null ? + resources.get(0).getVolumes().get(0).getAllocatedSizeKib() : 0; + kvmDisk.setSize(allocatedKib >= 0 ? allocatedKib * 1024 : 0); + kvmDisk.setVirtualSize(size); return kvmDisk; } else { logger.error("Linstor: viewResources didn't return resources or volumes."); @@ -410,7 +421,7 @@ public class LinstorStorageAdaptor implements StorageAdaptor { if (rsc.getFlags() != null && rsc.getFlags().contains(ApiConsts.FLAG_DRBD_DISKLESS) && !rsc.getFlags().contains(ApiConsts.FLAG_TIE_BREAKER)) { - ApiCallRcList delAnswers = api.resourceDelete(rsc.getName(), localNodeName); + ApiCallRcList delAnswers = api.resourceDelete(rsc.getName(), localNodeName, true); logLinstorAnswers(delAnswers); } } catch (ApiException apiEx) { @@ -473,21 +484,56 @@ public class LinstorStorageAdaptor implements StorageAdaptor { return false; } + /** + * Decrements the aux property key for template resource and deletes or just deletes if not template resource. + * @param api + * @param rscName + * @param rscGrpName + * @return + * @throws ApiException + */ + private boolean deRefOrDeleteResource(DevelopersApi api, String rscName, String rscGrpName) throws ApiException { + boolean deleted = false; + List existingRDs = LinstorUtil.getRDListStartingWith(api, rscName); + for (ResourceDefinition rd : existingRDs) { + int expectedProps = 0; // if it is a non template resource, we don't expect any _cs-template-for- prop + String propKey = LinstorUtil.getTemplateForAuxPropKey(rscGrpName); + if (rd.getProps().containsKey(propKey)) { + ResourceDefinitionModify rdm = new ResourceDefinitionModify(); + rdm.deleteProps(Collections.singletonList(propKey)); + api.resourceDefinitionModify(rd.getName(), rdm); + expectedProps = 1; + } + + // if there is only one template-for property left for templates, the template isn't needed anymore + // or if it isn't a template anyway, it will not have this Aux property + // _cs-template-for- properties work like a ref-count. + if (rd.getProps().keySet().stream() + .filter(key -> key.startsWith("Aux/" + LinstorUtil.CS_TEMPLATE_FOR_PREFIX)) + .count() == expectedProps) { + ApiCallRcList answers = api.resourceDefinitionDelete(rd.getName()); + checkLinstorAnswersThrow(answers); + deleted = true; + } + } + return deleted; + } + @Override public boolean deletePhysicalDisk(String name, KVMStoragePool pool, Storage.ImageFormat format) { logger.debug("Linstor: deletePhysicalDisk " + name); final DevelopersApi api = getLinstorAPI(pool); + final String rscName = getLinstorRscName(name); + final LinstorStoragePool linstorPool = (LinstorStoragePool) pool; + String rscGrpName = linstorPool.getResourceGroup(); try { - final String rscName = getLinstorRscName(name); - logger.debug("Linstor: delete resource definition " + rscName); - ApiCallRcList answers = api.resourceDefinitionDelete(rscName); - handleLinstorApiAnswers(answers, "Linstor: Unable to delete resource definition " + rscName); + return deRefOrDeleteResource(api, rscName, rscGrpName); } catch (ApiException apiEx) { + logger.error("Linstor: ApiEx - " + apiEx.getMessage()); throw new CloudRuntimeException(apiEx.getBestMessage(), apiEx); } - return true; } @Override @@ -561,6 +607,56 @@ public class LinstorStorageAdaptor implements StorageAdaptor { return false; } + /** + * Checks if the given disk is the SystemVM template, by checking its properties file in the same directory. + * The initial systemvm template resource isn't created on the management server, but + * we now need to know if the systemvm template is used, while copying. + * @param disk + * @return True if it is the systemvm template disk, else false. + */ + private static boolean isSystemTemplate(KVMPhysicalDisk disk) { + Path diskPath = Paths.get(disk.getPath()); + Path propFile = diskPath.getParent().resolve("template.properties"); + if (Files.exists(propFile)) { + java.util.Properties templateProps = new java.util.Properties(); + try { + templateProps.load(new FileInputStream(propFile.toFile())); + String desc = templateProps.getProperty("description"); + if (desc.startsWith("SystemVM Template")) { + return true; + } + } catch (IOException e) { + return false; + } + } + return false; + } + + /** + * Conditionally sets the correct aux properties for templates or basic resources. + * @param api + * @param srcDisk + * @param destPool + * @param name + */ + private void setRscDfnAuxProperties( + DevelopersApi api, KVMPhysicalDisk srcDisk, KVMStoragePool destPool, String name) { + // if it is the initial systemvm disk copy, we need to apply the _cs-template-for property. + if (isSystemTemplate(srcDisk)) { + applyAuxProps(api, name, "SystemVM Template", null); + LinstorStoragePool linPool = (LinstorStoragePool) destPool; + final String rscName = getLinstorRscName(name); + try { + LinstorUtil.setAuxTemplateForProperty(api, rscName, linPool.getResourceGroup()); + } catch (ApiException apiExc) { + logger.error("Error setting aux template for property for {}", rscName); + logLinstorAnswers(apiExc.getApiCallRcList()); + } + } else { + applyAuxProps(api, name, srcDisk.getDispName(), srcDisk.getVmName()); + } + } + @Override public KVMPhysicalDisk copyPhysicalDisk(KVMPhysicalDisk disk, String name, KVMStoragePool destPools, int timeout, byte[] srcPassphrase, byte[] destPassphrase, Storage.ProvisioningType provisioningType) { @@ -574,15 +670,14 @@ public class LinstorStorageAdaptor implements StorageAdaptor { name, QemuImg.PhysicalDiskFormat.RAW, provisioningType, disk.getVirtualSize(), null); final DevelopersApi api = getLinstorAPI(destPools); - applyAuxProps(api, name, disk.getDispName(), disk.getVmName()); + setRscDfnAuxProperties(api, disk, destPools, name); logger.debug("Linstor.copyPhysicalDisk: dstPath: {}", dstDisk.getPath()); final QemuImgFile destFile = new QemuImgFile(dstDisk.getPath()); destFile.setFormat(dstDisk.getFormat()); destFile.setSize(disk.getVirtualSize()); - boolean zeroedDevice = resourceSupportZeroBlocks(destPools, LinstorUtil.RSC_PREFIX + name); - + boolean zeroedDevice = resourceSupportZeroBlocks(destPools, getLinstorRscName(name)); try { final QemuImg qemu = new QemuImg(timeout, zeroedDevice, true); qemu.convert(srcFile, destFile); diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java index 8abd8fd8bd2..1afba442be9 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java @@ -21,11 +21,16 @@ import com.linbit.linstor.api.CloneWaiter; import com.linbit.linstor.api.DevelopersApi; import com.linbit.linstor.api.model.ApiCallRc; import com.linbit.linstor.api.model.ApiCallRcList; +import com.linbit.linstor.api.model.AutoSelectFilter; +import com.linbit.linstor.api.model.LayerType; import com.linbit.linstor.api.model.Properties; import com.linbit.linstor.api.model.ResourceDefinition; import com.linbit.linstor.api.model.ResourceDefinitionCloneRequest; import com.linbit.linstor.api.model.ResourceDefinitionCloneStarted; import com.linbit.linstor.api.model.ResourceDefinitionCreate; + +import com.linbit.linstor.api.model.ResourceDefinitionModify; +import com.linbit.linstor.api.model.ResourceGroup; import com.linbit.linstor.api.model.ResourceGroupSpawn; import com.linbit.linstor.api.model.ResourceMakeAvailable; import com.linbit.linstor.api.model.Snapshot; @@ -34,6 +39,7 @@ import com.linbit.linstor.api.model.VolumeDefinition; import com.linbit.linstor.api.model.VolumeDefinitionModify; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import javax.inject.Inject; import java.util.Arrays; @@ -43,6 +49,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.stream.Collectors; import com.cloud.agent.api.Answer; import com.cloud.agent.api.storage.ResizeVolumeAnswer; @@ -66,6 +73,7 @@ import com.cloud.storage.Storage.StoragePoolType; import com.cloud.storage.StorageManager; import com.cloud.storage.StoragePool; import com.cloud.storage.VMTemplateStoragePoolVO; +import com.cloud.storage.VMTemplateStorageResourceAssoc; import com.cloud.storage.Volume; import com.cloud.storage.VolumeDetailVO; import com.cloud.storage.VolumeVO; @@ -85,6 +93,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreCapabilities; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; @@ -93,6 +102,7 @@ import org.apache.cloudstack.framework.async.AsyncCompletionCallback; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.storage.RemoteHostEndPoint; import org.apache.cloudstack.storage.command.CommandResult; +import org.apache.cloudstack.storage.command.CopyCmdAnswer; import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.command.CreateObjectAnswer; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; @@ -103,8 +113,12 @@ import org.apache.cloudstack.storage.snapshot.SnapshotObject; import org.apache.cloudstack.storage.to.SnapshotObjectTO; import org.apache.cloudstack.storage.to.VolumeObjectTO; import org.apache.cloudstack.storage.volume.VolumeObject; + import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; +import org.apache.commons.collections.CollectionUtils; + +import java.nio.charset.StandardCharsets; public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver { protected Logger logger = LogManager.getLogger(getClass()); @@ -216,7 +230,7 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver } throw new CloudRuntimeException("Linstor: Unable to delete resource definition: " + rscDefName); } - logger.info(String.format("Linstor: Deleted resource %s", rscDefName)); + logger.info("Linstor: Deleted resource {}", rscDefName); } catch (ApiException apiEx) { logger.error("Linstor: ApiEx - " + apiEx.getMessage()); @@ -394,22 +408,166 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver storagePoolVO.getUserInfo() : "DfltRscGrp"; } - private String createResourceBase( - String rscName, long sizeInBytes, String volName, String vmName, DevelopersApi api, String rscGrp) { - ResourceGroupSpawn rscGrpSpawn = new ResourceGroupSpawn(); - rscGrpSpawn.setResourceDefinitionName(rscName); - rscGrpSpawn.addVolumeSizesItem(sizeInBytes / 1024); + /** + * Returns the layerlist of the resourceGroup with encryption(LUKS) added above STORAGE. + * If the resourceGroup layer list already contains LUKS this layer list will be returned. + * @param api Linstor developers API + * @param resourceGroup Resource group to get the encryption layer list + * @return layer list with LUKS added + */ + public List getEncryptedLayerList(DevelopersApi api, String resourceGroup) { + try { + List rscGrps = api.resourceGroupList( + Collections.singletonList(resourceGroup), Collections.emptyList(), null, null); + if (CollectionUtils.isEmpty(rscGrps)) { + throw new CloudRuntimeException( + String.format("Resource Group %s not found on Linstor cluster.", resourceGroup)); + } + + final ResourceGroup rscGrp = rscGrps.get(0); + List layers = Arrays.asList(LayerType.DRBD, LayerType.LUKS, LayerType.STORAGE); + List curLayerStack = rscGrp.getSelectFilter() != null ? + rscGrp.getSelectFilter().getLayerStack() : Collections.emptyList(); + if (CollectionUtils.isNotEmpty(curLayerStack)) { + layers = curLayerStack.stream().map(LayerType::valueOf).collect(Collectors.toList()); + if (!layers.contains(LayerType.LUKS)) { + layers.add(layers.size() - 1, LayerType.LUKS); // lowest layer is STORAGE + } + } + return layers; + } catch (ApiException e) { + throw new CloudRuntimeException( + String.format("Resource Group %s not found on Linstor cluster.", resourceGroup)); + } + } + + /** + * Spawns a new Linstor resource with the given arguments. + * @param api + * @param newRscName + * @param sizeInBytes + * @param isTemplate + * @param rscGrpName + * @param volName + * @param vmName + * @throws ApiException + */ + private void spawnResource( + DevelopersApi api, String newRscName, long sizeInBytes, boolean isTemplate, String rscGrpName, + String volName, String vmName, @Nullable Long passPhraseId, @Nullable byte[] passPhrase) throws ApiException + { + ResourceGroupSpawn rscGrpSpawn = new ResourceGroupSpawn(); + rscGrpSpawn.setResourceDefinitionName(newRscName); + rscGrpSpawn.addVolumeSizesItem(sizeInBytes / 1024); + if (passPhraseId != null) { + AutoSelectFilter asf = new AutoSelectFilter(); + List luksLayers = getEncryptedLayerList(api, rscGrpName); + asf.setLayerStack(luksLayers.stream().map(LayerType::toString).collect(Collectors.toList())); + rscGrpSpawn.setSelectFilter(asf); + if (passPhrase != null) { + String utf8Passphrase = new String(passPhrase, StandardCharsets.UTF_8); + rscGrpSpawn.setVolumePassphrases(Collections.singletonList(utf8Passphrase)); + } + } + + if (isTemplate) { + Properties props = new Properties(); + props.put(LinstorUtil.getTemplateForAuxPropKey(rscGrpName), "true"); + rscGrpSpawn.setResourceDefinitionProps(props); + } + + logger.info("Linstor: Spawn resource " + newRscName); + ApiCallRcList answers = api.resourceGroupSpawn(rscGrpName, rscGrpSpawn); + checkLinstorAnswersThrow(answers); + + answers = LinstorUtil.applyAuxProps(api, newRscName, volName, vmName); + checkLinstorAnswersThrow(answers); + } + + /** + * Condition if a template resource can be shared with the given resource group. + * @param tgtRscGrp + * @param tgtLayerStack + * @param rg + * @return True if the template resource can be shared, else false. + */ + private boolean canShareTemplateForResourceGroup( + ResourceGroup tgtRscGrp, List tgtLayerStack, ResourceGroup rg) { + List rgLayerStack = rg.getSelectFilter() != null ? + rg.getSelectFilter().getLayerStack() : null; + return Objects.equals(tgtLayerStack, rgLayerStack) && + Objects.equals(tgtRscGrp.getSelectFilter().getStoragePoolList(), + rg.getSelectFilter().getStoragePoolList()); + } + + /** + * Searches for a shareable template for this rscGrpName and sets the aux template property. + * @param api + * @param rscName + * @param rscGrpName + * @param existingRDs + * @return + * @throws ApiException + */ + private boolean foundShareableTemplate( + DevelopersApi api, String rscName, String rscGrpName, + List> existingRDs) throws ApiException { + if (!existingRDs.isEmpty()) { + ResourceGroup tgtRscGrp = api.resourceGroupList( + Collections.singletonList(rscGrpName), null, null, null).get(0); + List tgtLayerStack = tgtRscGrp.getSelectFilter() != null ? + tgtRscGrp.getSelectFilter().getLayerStack() : null; + + // check if there is already a template copy, that we could reuse + // this means if select filters are similar enough to allow cloning from + for (Pair rdPair : existingRDs) { + ResourceGroup rg = rdPair.second(); + if (canShareTemplateForResourceGroup(tgtRscGrp, tgtLayerStack, rg)) { + LinstorUtil.setAuxTemplateForProperty(api, rscName, rscGrpName); + return true; + } + } + } + return false; + } + + /** + * Creates a new Linstor resource. + * @param rscName + * @param sizeInBytes + * @param volName + * @param vmName + * @param api + * @param rscGrp + * @param poolId + * @param isTemplate indicates if the resource is a template + * @return true if a new resource was created, false if it already existed or was reused. + */ + private boolean createResourceBase( + String rscName, long sizeInBytes, String volName, String vmName, + @Nullable Long passPhraseId, @Nullable byte[] passPhrase, DevelopersApi api, + String rscGrp, long poolId, boolean isTemplate) + { try { - logger.info("Linstor: Spawn resource " + rscName); - ApiCallRcList answers = api.resourceGroupSpawn(rscGrp, rscGrpSpawn); - checkLinstorAnswersThrow(answers); + logger.debug("createRscBase: {} :: {} :: {}", rscName, rscGrp, isTemplate); + List> existingRDs = LinstorUtil.getRDAndRGListStartingWith(api, rscName); - answers = LinstorUtil.applyAuxProps(api, rscName, volName, vmName); - checkLinstorAnswersThrow(answers); - - return LinstorUtil.getDevicePath(api, rscName); + String fullRscName = String.format("%s-%d", rscName, poolId); + boolean alreadyCreated = existingRDs.stream() + .anyMatch(p -> p.first().getName().equalsIgnoreCase(fullRscName)) || + existingRDs.stream().anyMatch(p -> p.first().getProps().containsKey(LinstorUtil.getTemplateForAuxPropKey(rscGrp))); + if (!alreadyCreated) { + boolean createNewRsc = !foundShareableTemplate(api, rscName, rscGrp, existingRDs); + if (createNewRsc) { + String newRscName = existingRDs.isEmpty() ? rscName : fullRscName; + spawnResource(api, newRscName, sizeInBytes, isTemplate, rscGrp, + volName, vmName, passPhraseId, passPhrase); + } + return createNewRsc; + } + return false; } catch (ApiException apiEx) { logger.error("Linstor: ApiEx - " + apiEx.getMessage()); @@ -422,8 +580,9 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver final String rscGrp = getRscGrp(storagePoolVO); final String rscName = LinstorUtil.RSC_PREFIX + vol.getUuid(); - String deviceName = createResourceBase( - rscName, vol.getSize(), vol.getName(), vol.getAttachedVmName(), linstorApi, rscGrp); + createResourceBase( + rscName, vol.getSize(), vol.getName(), vol.getAttachedVmName(), vol.getPassphraseId(), vol.getPassphrase(), + linstorApi, rscGrp, storagePoolVO.getId(), false); try { @@ -450,20 +609,83 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver } } + /** + * Update resource-definitions resource-group to the correct one if it isn't already the intended. + * @param api Linstor api + * @param rscName resource name to check the resource group + * @param tgtRscGrp resource group name to set + * @throws ApiException exception if any api error occurred + */ + private void updateRscGrpIfNecessary(DevelopersApi api, String rscName, String tgtRscGrp) throws ApiException { + List rscDfns = api.resourceDefinitionList( + Collections.singletonList(rscName), null, null, null); + if (rscDfns != null && !rscDfns.isEmpty()) { + ResourceDefinition rscDfn = rscDfns.get(0); + + if (!rscDfn.getResourceGroupName().equalsIgnoreCase(tgtRscGrp)) { + ResourceDefinitionModify rdm = new ResourceDefinitionModify(); + rdm.setResourceGroup(tgtRscGrp); + ApiCallRcList answers = api.resourceDefinitionModify(rscName, rdm); + + if (answers.hasError()) { + String bestError = LinstorUtil.getBestErrorMessage(answers); + logger.error("Update resource group on {} error: {}", rscName, bestError); + throw new CloudRuntimeException(bestError); + } else { + logger.info("Successfully changed resource group to {} on {}", tgtRscGrp, rscName); + } + } + } + } + + /** + * If a resource is cloned, all properties are cloned too, but the _cs-template-for properties, + * should only stay on the template resource, so delete them in this method. + * @param api + * @param rscName + * @throws ApiException + */ + private void deleteTemplateForProps( + DevelopersApi api, String rscName) throws ApiException { + List rdList = api.resourceDefinitionList( + Collections.singletonList(rscName), null, null, null); + + if (CollectionUtils.isNotEmpty(rdList)) { + ResourceDefinitionModify rdm = new ResourceDefinitionModify(); + List deleteProps = rdList.get(0).getProps().keySet().stream() + .filter(key -> key.startsWith("Aux/" + LinstorUtil.CS_TEMPLATE_FOR_PREFIX)) + .collect(Collectors.toList()); + rdm.deleteProps(deleteProps); + ApiCallRcList answers = api.resourceDefinitionModify(rscName, rdm); + checkLinstorAnswers(answers); + } + } + private String cloneResource(long csCloneId, VolumeInfo volumeInfo, StoragePoolVO storagePoolVO) { // get the cached template on this storage VMTemplateStoragePoolVO tmplPoolRef = _vmTemplatePoolDao.findByPoolTemplate( storagePoolVO.getId(), csCloneId, null); if (tmplPoolRef != null) { - final String cloneRes = LinstorUtil.RSC_PREFIX + tmplPoolRef.getLocalDownloadPath(); + final String templateRscName = LinstorUtil.RSC_PREFIX + tmplPoolRef.getLocalDownloadPath(); final String rscName = LinstorUtil.RSC_PREFIX + volumeInfo.getUuid(); final DevelopersApi linstorApi = LinstorUtil.getLinstorAPI(storagePoolVO.getHostAddress()); try { - logger.info("Clone resource definition " + cloneRes + " to " + rscName); + ResourceDefinition templateRD = LinstorUtil.findResourceDefinition( + linstorApi, templateRscName, getRscGrp(storagePoolVO)); + final String cloneRes = templateRD != null ? templateRD.getName() : templateRscName; + logger.info("Clone resource definition {} to {}", cloneRes, rscName); ResourceDefinitionCloneRequest cloneRequest = new ResourceDefinitionCloneRequest(); cloneRequest.setName(rscName); + if (volumeInfo.getPassphraseId() != null) { + List encryptionLayer = getEncryptedLayerList(linstorApi, getRscGrp(storagePoolVO)); + cloneRequest.setLayerList(encryptionLayer); + if (volumeInfo.getPassphrase() != null) { + String utf8Passphrase = new String(volumeInfo.getPassphrase(), StandardCharsets.UTF_8); + cloneRequest.setVolumePassphrases(Collections.singletonList(utf8Passphrase)); + } + } ResourceDefinitionCloneStarted cloneStarted = linstorApi.resourceDefinitionClone( cloneRes, cloneRequest); @@ -479,6 +701,9 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver resizeResource(linstorApi, rscName, volumeInfo.getSize()); } + updateRscGrpIfNecessary(linstorApi, rscName, getRscGrp(storagePoolVO)); + + deleteTemplateForProps(linstorApi, rscName); LinstorUtil.applyAuxProps(linstorApi, rscName, volumeInfo.getName(), volumeInfo.getAttachedVmName()); applyQoSSettings(storagePoolVO, linstorApi, rscName, volumeInfo.getMaxIops()); @@ -906,41 +1131,74 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver return LinstorUtil.getDevicePath(api, restoredName); } + /** + * Updates the template_spool_ref DB entry to indicate that this template was fully downloaded and is ready. + * @param templateId + * @param destTemplateInfoUuid + * @param destDataStoreId + * @param templateSize + */ + private void updateTemplateSpoolRef( + long templateId, String destTemplateInfoUuid, long destDataStoreId, long templateSize) { + VMTemplateStoragePoolVO destVolumeTemplateStoragePoolVO = _vmTemplatePoolDao.findByPoolTemplate( + destDataStoreId, templateId, null); + if (destVolumeTemplateStoragePoolVO == null) { + throw new CloudRuntimeException( + String.format("Unable to find template_spool_ref entry for pool_id %d and template_id %d", + destDataStoreId, templateId)); + } + destVolumeTemplateStoragePoolVO.setDownloadPercent(100); + destVolumeTemplateStoragePoolVO.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOADED); + destVolumeTemplateStoragePoolVO.setState(ObjectInDataStoreStateMachine.State.Ready); + destVolumeTemplateStoragePoolVO.setTemplateSize(templateSize); + destVolumeTemplateStoragePoolVO.setLocalDownloadPath(destTemplateInfoUuid); + destVolumeTemplateStoragePoolVO.setInstallPath(destTemplateInfoUuid); + _vmTemplatePoolDao.persist(destVolumeTemplateStoragePoolVO); + } + private Answer copyTemplate(DataObject srcData, DataObject dstData) { TemplateInfo tInfo = (TemplateInfo) dstData; final StoragePoolVO pool = _storagePoolDao.findById(dstData.getDataStore().getId()); final DevelopersApi api = LinstorUtil.getLinstorAPI(pool.getHostAddress()); final String rscName = LinstorUtil.RSC_PREFIX + dstData.getUuid(); - createResourceBase( + boolean newCreated = createResourceBase( LinstorUtil.RSC_PREFIX + dstData.getUuid(), tInfo.getSize(), tInfo.getName(), "", + null, + null, api, - getRscGrp(pool)); + getRscGrp(pool), + pool.getId(), + true); - int nMaxExecutionMinutes = NumbersUtil.parseInt( - _configDao.getValue(Config.SecStorageCmdExecutionTimeMax.key()), 30); - CopyCommand cmd = new CopyCommand( - srcData.getTO(), - dstData.getTO(), - nMaxExecutionMinutes * 60 * 1000, - VirtualMachineManager.ExecuteInSequence.value()); Answer answer; + if (newCreated) { + int nMaxExecutionMinutes = NumbersUtil.parseInt( + _configDao.getValue(Config.SecStorageCmdExecutionTimeMax.key()), 30); + CopyCommand cmd = new CopyCommand( + srcData.getTO(), + dstData.getTO(), + nMaxExecutionMinutes * 60 * 1000, + VirtualMachineManager.ExecuteInSequence.value()); - try { - Optional optEP = getLinstorEP(api, rscName); - if (optEP.isPresent()) { - answer = optEP.get().sendMessage(cmd); - } - else { - answer = new Answer(cmd, false, "Unable to get matching Linstor endpoint."); + try { + Optional optEP = getLinstorEP(api, rscName); + if (optEP.isPresent()) { + answer = optEP.get().sendMessage(cmd); + } else { + answer = new Answer(cmd, false, "Unable to get matching Linstor endpoint."); + deleteResourceDefinition(pool, rscName); + } + } catch (ApiException exc) { + logger.error("copy template failed: ", exc); deleteResourceDefinition(pool, rscName); + throw new CloudRuntimeException(exc.getBestMessage()); } - } catch (ApiException exc) { - logger.error("copy template failed: ", exc); - deleteResourceDefinition(pool, rscName); - throw new CloudRuntimeException(exc.getBestMessage()); + } else { + updateTemplateSpoolRef(dstData.getId(), tInfo.getUuid(), dstData.getDataStore().getId(), srcData.getSize()); + answer = new Answer(new CopyCmdAnswer(dstData.getTO())); } return answer; } diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java index cbdc9940b80..87b31d70554 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java @@ -26,6 +26,7 @@ import com.linbit.linstor.api.model.Node; import com.linbit.linstor.api.model.Properties; import com.linbit.linstor.api.model.ProviderKind; import com.linbit.linstor.api.model.Resource; +import com.linbit.linstor.api.model.ResourceDefinition; import com.linbit.linstor.api.model.ResourceDefinitionModify; import com.linbit.linstor.api.model.ResourceGroup; import com.linbit.linstor.api.model.ResourceWithVolumes; @@ -37,8 +38,11 @@ import javax.annotation.Nonnull; import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; +import com.cloud.utils.Pair; import com.cloud.utils.exception.CloudRuntimeException; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; @@ -49,6 +53,7 @@ public class LinstorUtil { public final static String PROVIDER_NAME = "Linstor"; public static final String RSC_PREFIX = "cs-"; public static final String RSC_GROUP = "resourceGroup"; + public static final String CS_TEMPLATE_FOR_PREFIX = "_cs-template-for-"; public static final String TEMP_VOLUME_ID = "tempVolumeId"; @@ -288,4 +293,114 @@ public class LinstorUtil { } return answers; } + + /** + * Returns all resource definitions that start with the given `startWith` name. + * @param api + * @param startWith startWith String + * @return a List with all ResourceDefinition starting with `startWith` + * @throws ApiException + */ + public static List getRDListStartingWith(DevelopersApi api, String startWith) + throws ApiException + { + List rscDfns = api.resourceDefinitionList(null, null, null, null); + + return rscDfns.stream() + .filter(rscDfn -> rscDfn.getName().toLowerCase().startsWith(startWith.toLowerCase())) + .collect(Collectors.toList()); + } + + /** + * Returns a pair list of resource-definitions with ther 1:1 mapped resource-group objects that start with the + * resource name `startWith` + * @param api + * @param startWith + * @return + * @throws ApiException + */ + public static List> getRDAndRGListStartingWith(DevelopersApi api, String startWith) + throws ApiException + { + List foundRDs = getRDListStartingWith(api, startWith); + + List rscGrpStrings = foundRDs.stream() + .map(ResourceDefinition::getResourceGroupName) + .collect(Collectors.toList()); + + Map rscGrps = api.resourceGroupList(rscGrpStrings, null, null, null).stream() + .collect(Collectors.toMap(ResourceGroup::getName, rscGrp -> rscGrp)); + + return foundRDs.stream() + .map(rd -> new Pair<>(rd, rscGrps.get(rd.getResourceGroupName()))) + .collect(Collectors.toList()); + } + + /** + * The full name of template-for aux property key. + * @param rscGrpName + * @return + */ + public static String getTemplateForAuxPropKey(String rscGrpName) { + return String.format("Aux/%s%s", CS_TEMPLATE_FOR_PREFIX, rscGrpName); + } + + /** + * Template resource should have a _cs-template-for-... property, that indicates to which resource-group + * this template belongs, it works like a refcount to keep it alive if there are still such properties on the + * template resource. That methods set the correct property on the given resource. + * @param api + * @param rscName Resource name to set the property. + * @param rscGrpName Resource group this template should belong too. + * @throws ApiException + */ + public static void setAuxTemplateForProperty(DevelopersApi api, String rscName, String rscGrpName) + throws ApiException + { + ResourceDefinitionModify rdm = new ResourceDefinitionModify(); + Properties props = new Properties(); + String propKey = LinstorUtil.getTemplateForAuxPropKey(rscGrpName); + props.put(propKey, "true"); + rdm.setOverrideProps(props); + ApiCallRcList answers = api.resourceDefinitionModify(rscName, rdm); + + if (answers.hasError()) { + String bestError = LinstorUtil.getBestErrorMessage(answers); + LOGGER.error("Set {} on {} error: {}", propKey, rscName, bestError); + throw new CloudRuntimeException(bestError); + } else { + LOGGER.info("Set {} property on {}", propKey, rscName); + } + } + + /** + * Find the correct resource definition to clone from. + * There could be multiple resource definitions for the same template, with the same prefix. + * This method searches for which resource group the resource definition was intended and returns that. + * If no exact resource definition could be found, we return the first with a similar name as a fallback. + * If there is not even one with the correct prefix, we return null. + * @param api + * @param rscName + * @param rscGrpName + * @return The resource-definition to clone from, if no template and no match, return null. + * @throws ApiException + */ + public static ResourceDefinition findResourceDefinition(DevelopersApi api, String rscName, String rscGrpName) + throws ApiException { + List rscDfns = api.resourceDefinitionList(null, null, null, null); + + List rdsStartingWith = rscDfns.stream() + .filter(rscDfn -> rscDfn.getName().toLowerCase().startsWith(rscName.toLowerCase())) + .collect(Collectors.toList()); + + if (rdsStartingWith.isEmpty()) { + return null; + } + + Optional rd = rdsStartingWith.stream() + .filter(rscDfn -> rscDfn.getProps().containsKey(LinstorUtil.getTemplateForAuxPropKey(rscGrpName))) + .findFirst(); + + return rd.orElseGet(() -> rdsStartingWith.get(0)); + } } diff --git a/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImplTest.java b/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImplTest.java new file mode 100644 index 00000000000..75276739468 --- /dev/null +++ b/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImplTest.java @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.datastore.driver; + +import com.linbit.linstor.api.ApiException; +import com.linbit.linstor.api.DevelopersApi; +import com.linbit.linstor.api.model.AutoSelectFilter; +import com.linbit.linstor.api.model.LayerType; +import com.linbit.linstor.api.model.ResourceGroup; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.junit.MockitoJUnitRunner; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class LinstorPrimaryDataStoreDriverImplTest { + + private DevelopersApi api; + + @InjectMocks + private LinstorPrimaryDataStoreDriverImpl linstorPrimaryDataStoreDriver; + + @Before + public void setUp() { + api = mock(DevelopersApi.class); + } + + @Test + public void testGetEncryptedLayerList() throws ApiException { + ResourceGroup dfltRscGrp = new ResourceGroup(); + dfltRscGrp.setName("DfltRscGrp"); + + ResourceGroup bCacheRscGrp = new ResourceGroup(); + bCacheRscGrp.setName("BcacheGrp"); + AutoSelectFilter asf = new AutoSelectFilter(); + asf.setLayerStack(Arrays.asList(LayerType.DRBD.name(), LayerType.BCACHE.name(), LayerType.STORAGE.name())); + asf.setStoragePool("nvmePool"); + bCacheRscGrp.setSelectFilter(asf); + + ResourceGroup encryptedGrp = new ResourceGroup(); + encryptedGrp.setName("EncryptedGrp"); + AutoSelectFilter asf2 = new AutoSelectFilter(); + asf2.setLayerStack(Arrays.asList(LayerType.DRBD.name(), LayerType.LUKS.name(), LayerType.STORAGE.name())); + asf2.setStoragePool("ssdPool"); + encryptedGrp.setSelectFilter(asf2); + + when(api.resourceGroupList(Collections.singletonList("DfltRscGrp"), Collections.emptyList(), null, null)) + .thenReturn(Collections.singletonList(dfltRscGrp)); + when(api.resourceGroupList(Collections.singletonList("BcacheGrp"), Collections.emptyList(), null, null)) + .thenReturn(Collections.singletonList(bCacheRscGrp)); + when(api.resourceGroupList(Collections.singletonList("EncryptedGrp"), Collections.emptyList(), null, null)) + .thenReturn(Collections.singletonList(encryptedGrp)); + + List layers = linstorPrimaryDataStoreDriver.getEncryptedLayerList(api, "DfltRscGrp"); + Assert.assertEquals(Arrays.asList(LayerType.DRBD, LayerType.LUKS, LayerType.STORAGE), layers); + + layers = linstorPrimaryDataStoreDriver.getEncryptedLayerList(api, "BcacheGrp"); + Assert.assertEquals(Arrays.asList(LayerType.DRBD, LayerType.BCACHE, LayerType.LUKS, LayerType.STORAGE), layers); + + layers = linstorPrimaryDataStoreDriver.getEncryptedLayerList(api, "EncryptedGrp"); + Assert.assertEquals(Arrays.asList(LayerType.DRBD, LayerType.LUKS, LayerType.STORAGE), layers); + } +} diff --git a/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/util/LinstorUtilTest.java b/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/util/LinstorUtilTest.java new file mode 100644 index 00000000000..55f0c6ebe6d --- /dev/null +++ b/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/util/LinstorUtilTest.java @@ -0,0 +1,127 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.datastore.util; + +import com.linbit.linstor.api.ApiException; +import com.linbit.linstor.api.DevelopersApi; +import com.linbit.linstor.api.model.AutoSelectFilter; +import com.linbit.linstor.api.model.Node; +import com.linbit.linstor.api.model.Properties; +import com.linbit.linstor.api.model.ProviderKind; +import com.linbit.linstor.api.model.ResourceGroup; +import com.linbit.linstor.api.model.StoragePool; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class LinstorUtilTest { + + private static final String LINSTOR_URL_TEST = "devnull.com:3370"; + private DevelopersApi api; + + private Node mockNode(String name) { + Node nodeMock = new Node(); + nodeMock.setName(name); + + return nodeMock; + } + + private StoragePool mockStoragePool(String name, String node, ProviderKind kind) { + StoragePool sp = new StoragePool(); + sp.setStoragePoolName(name); + sp.setNodeName(node); + sp.setProviderKind(kind); + return sp; + } + + @Before + public void setUp() throws ApiException { + api = mock(DevelopersApi.class); + + when(api.nodeList(Collections.emptyList(), Collections.emptyList(), null, null)) + .thenReturn(Arrays.asList(mockNode("nodeA"), mockNode("nodeB"), mockNode("nodeC"))); + + ResourceGroup csGroup = new ResourceGroup(); + csGroup.setName("cloudstack"); + AutoSelectFilter asf = new AutoSelectFilter(); + asf.setPlaceCount(2); + csGroup.setSelectFilter(asf); + when(api.resourceGroupList(Collections.singletonList("cloudstack"), null, null, null)) + .thenReturn(Collections.singletonList(csGroup)); + + when(api.viewStoragePools(Collections.emptyList(), null, null, null, null, true)) + .thenReturn(Arrays.asList( + mockStoragePool("thinpool", "nodeA", ProviderKind.LVM_THIN), + mockStoragePool("thinpool", "nodeB", ProviderKind.LVM_THIN), + mockStoragePool("thinpool", "nodeC", ProviderKind.LVM_THIN) + )); + +// when(LinstorUtil.getLinstorAPI(LINSTOR_URL_TEST)).thenReturn(api); + } + + @Test + public void testGetLinstorNodeNames() throws ApiException { + List linstorNodes = LinstorUtil.getLinstorNodeNames(api); + Assert.assertEquals(Arrays.asList("nodeA", "nodeB", "nodeC"), linstorNodes); + } + + @Test + public void testGetSnapshotPath() { + { + StoragePool spLVMThin = new StoragePool(); + Properties lvmThinProps = new Properties(); + lvmThinProps.put("StorDriver/StorPoolName", "storage/storage-thin"); + spLVMThin.setProps(lvmThinProps); + spLVMThin.setProviderKind(ProviderKind.LVM_THIN); + String snapPath = LinstorUtil.getSnapshotPath(spLVMThin, "cs-cb32532a-dd8f-47e0-a81c-8a75573d3545", "snap3"); + Assert.assertEquals("/dev/mapper/storage-cs--cb32532a--dd8f--47e0--a81c--8a75573d3545_00000_snap3", snapPath); + } + + { + StoragePool spZFS = new StoragePool(); + Properties zfsProps = new Properties(); + zfsProps.put("StorDriver/StorPoolName", "linstorPool"); + spZFS.setProps(zfsProps); + spZFS.setProviderKind(ProviderKind.ZFS); + + String snapPath = LinstorUtil.getSnapshotPath(spZFS, "cs-cb32532a-dd8f-47e0-a81c-8a75573d3545", "snap2"); + Assert.assertEquals("zfs://linstorPool/cs-cb32532a-dd8f-47e0-a81c-8a75573d3545_00000@snap2", snapPath); + } + } + + @Test + public void testGetRscGroupStoragePools() throws ApiException { + List storagePools = LinstorUtil.getRscGroupStoragePools(api, "cloudstack"); + + List names = storagePools.stream() + .map(sp -> String.format("%s::%s", sp.getNodeName(), sp.getStoragePoolName())) + .collect(Collectors.toList()); + Assert.assertEquals(names, Arrays.asList("nodeA::thinpool", "nodeB::thinpool", "nodeC::thinpool")); + } +} diff --git a/plugins/storage/volume/primera/src/main/java/org/apache/cloudstack/storage/datastore/adapter/primera/PrimeraAdapter.java b/plugins/storage/volume/primera/src/main/java/org/apache/cloudstack/storage/datastore/adapter/primera/PrimeraAdapter.java index 2c0fccc2131..445799f0c04 100644 --- a/plugins/storage/volume/primera/src/main/java/org/apache/cloudstack/storage/datastore/adapter/primera/PrimeraAdapter.java +++ b/plugins/storage/volume/primera/src/main/java/org/apache/cloudstack/storage/datastore/adapter/primera/PrimeraAdapter.java @@ -146,16 +146,18 @@ public class PrimeraAdapter implements ProviderAdapter { } // determine volume type based on offering - // THIN: tpvv=true, reduce=false - // SPARSE: tpvv=true, reduce=true - // THICK: tpvv=false, tpZeroFill=true (not supported) + // tpvv -- thin provisioned virtual volume (no deduplication) + // reduce -- thin provisioned virtual volume (with duplication and compression, also known as DECO) + // these are the only choices with newer Primera devices + // we will use THIN for the deduplicated/compressed type and SPARSE for thin-only without dedup/compress + // note: DECO/reduce type must be at least 16GB in size if (diskOffering != null) { if (diskOffering.getType() == ProvisioningType.THIN) { - request.setTpvv(true); - request.setReduce(false); - } else if (diskOffering.getType() == ProvisioningType.SPARSE) { request.setTpvv(false); request.setReduce(true); + } else if (diskOffering.getType() == ProvisioningType.SPARSE) { + request.setTpvv(true); + request.setReduce(false); } else if (diskOffering.getType() == ProvisioningType.FAT) { throw new RuntimeException("This storage provider does not support FAT provisioned volumes"); } @@ -166,8 +168,16 @@ public class PrimeraAdapter implements ProviderAdapter { } } else { // default to deduplicated volume - request.setReduce(true); request.setTpvv(false); + request.setReduce(true); + } + + if (request.getReduce() == true) { + // check if sizeMiB is less than 16GB adjust up to 16GB. The AdaptiveDatastoreDriver will automatically + // update this on the cloudstack side to match + if (request.getSizeMiB() < 16 * 1024) { + request.setSizeMiB(16 * 1024); + } } request.setComment(ProviderVolumeNamer.generateObjectComment(context, dataIn)); @@ -185,8 +195,11 @@ public class PrimeraAdapter implements ProviderAdapter { if (host == null) { throw new RuntimeException("Unable to find host " + hostname + " on storage provider"); } - request.setHostname(host.getName()); + // check if we already have a vlun for requested host + Integer vlun = hasVlun(hostname, hostname); + if (vlun == null) { + request.setHostname(host.getName()); request.setVolumeName(dataIn.getExternalName()); request.setAutoLun(true); // auto-lun returned here: Location: /api/v1/vluns/test_vv02,252,mysystem,2:2:4 @@ -198,7 +211,13 @@ public class PrimeraAdapter implements ProviderAdapter { if (toks.length <2) { throw new RuntimeException("Attach volume failed with invalid location response to vlun add command on storage provider. Provided location: " + location); } - return toks[1]; + try { + vlun = Integer.parseInt(toks[1]); + } catch (NumberFormatException e) { + throw new RuntimeException("VLUN attach request succeeded but the VLUN value is not a valid number: " + toks[1]); + } + } + return vlun.toString(); } /** @@ -233,6 +252,20 @@ public class PrimeraAdapter implements ProviderAdapter { } } + private Integer hasVlun(String externalName, String hostname) { + PrimeraVlunList list = getVluns(externalName); + if (list != null && list.getMembers().size() > 0) { + for (PrimeraVlun vlun: list.getMembers()) { + if (hostname != null) { + if (vlun.getHostname().equals(hostname) || vlun.getHostname().equals(hostname.split("\\.")[0])) { + return vlun.getLun(); + } + } + } + } + return null; + } + public void removeVlun(String name, Integer lunid, String hostString) { // hostString can be a hostname OR "set:". It is stored this way // in the appliance and returned as the vlun's name/string. diff --git a/pom.xml b/pom.xml index fb4d2429406..7168e3e8830 100644 --- a/pom.xml +++ b/pom.xml @@ -175,7 +175,7 @@ 10.1 2.6.6 0.6.0 - 0.5.2 + 0.6.0 0.10.2 3.4.4_1 4.0.1 diff --git a/scripts/storage/multipath/cleanStaleMaps.sh b/scripts/storage/multipath/cleanStaleMaps.sh index 90b9bef5a8d..c1ded42943c 100755 --- a/scripts/storage/multipath/cleanStaleMaps.sh +++ b/scripts/storage/multipath/cleanStaleMaps.sh @@ -22,10 +22,18 @@ # ############################################################################################# +SCRIPT_NAME=$(basename "$0") + +if [[ $(pgrep -f ${SCRIPT_NAME}) != "$$" ]]; then + echo "Another instance of ${SCRIPT_NAME} is already running! Exiting" + exit +fi + + cd $(dirname $0) for WWID in $(multipathd list maps status | awk '{ if ($4 == 0) { print substr($1,2); }}'); do - ./removeVolume.sh ${WWID} + ./disconnectVolume.sh ${WWID} done exit 0 diff --git a/scripts/storage/multipath/disconnectVolume.sh b/scripts/storage/multipath/disconnectVolume.sh index 067e561f8a3..f894076927f 100755 --- a/scripts/storage/multipath/disconnectVolume.sh +++ b/scripts/storage/multipath/disconnectVolume.sh @@ -66,6 +66,9 @@ fi logger -t CS_SCSI_VOL_REMOVE "${WWID} successfully purged from multipath along with slave devices" +# Added to give time for the event to be fired to the server +sleep 10 + echo "$(date): ${WWID} removed" exit 0 diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index 44cf148ca53..1f964fbbd42 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -590,7 +590,7 @@ public class ApiResponseHelper implements ResponseGenerator { } resourceLimitResponse.setResourceType(limit.getType()); - if ((limit.getType() == ResourceType.primary_storage || limit.getType() == ResourceType.secondary_storage) && limit.getMax() >= 0) { + if (ResourceType.isStorageType(limit.getType()) && limit.getMax() >= 0) { resourceLimitResponse.setMax((long)Math.ceil((double)limit.getMax() / ResourceType.bytesToGiB)); } else { resourceLimitResponse.setMax(limit.getMax()); diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java index f1b0dc795ac..8eebb55dce9 100644 --- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java @@ -356,263 +356,263 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q private static final String ID_FIELD = "id"; @Inject - private AccountManager accountMgr; + AccountManager accountMgr; @Inject - private ProjectManager _projectMgr; + ProjectManager _projectMgr; @Inject - private DomainDao _domainDao; + DomainDao _domainDao; @Inject - private DomainJoinDao _domainJoinDao; + DomainJoinDao _domainJoinDao; @Inject - private UserAccountJoinDao _userAccountJoinDao; + UserAccountJoinDao _userAccountJoinDao; @Inject - private EventDao eventDao; + EventDao eventDao; @Inject - private EventJoinDao _eventJoinDao; + EventJoinDao _eventJoinDao; @Inject - private ResourceTagJoinDao _resourceTagJoinDao; + ResourceTagJoinDao _resourceTagJoinDao; @Inject - private InstanceGroupJoinDao _vmGroupJoinDao; + InstanceGroupJoinDao _vmGroupJoinDao; @Inject - private UserVmJoinDao _userVmJoinDao; + UserVmJoinDao _userVmJoinDao; @Inject - private UserVmDao userVmDao; + UserVmDao userVmDao; @Inject - private VMInstanceDao _vmInstanceDao; + VMInstanceDao _vmInstanceDao; @Inject - private SecurityGroupJoinDao _securityGroupJoinDao; + SecurityGroupJoinDao _securityGroupJoinDao; @Inject - private SecurityGroupVMMapDao securityGroupVMMapDao; + SecurityGroupVMMapDao securityGroupVMMapDao; @Inject - private DomainRouterJoinDao _routerJoinDao; + DomainRouterJoinDao _routerJoinDao; @Inject - private ProjectInvitationJoinDao _projectInvitationJoinDao; + ProjectInvitationJoinDao _projectInvitationJoinDao; @Inject - private ProjectJoinDao _projectJoinDao; + ProjectJoinDao _projectJoinDao; @Inject - private ProjectDao _projectDao; + ProjectDao _projectDao; @Inject - private ProjectAccountDao _projectAccountDao; + ProjectAccountDao _projectAccountDao; @Inject - private ProjectAccountJoinDao _projectAccountJoinDao; + ProjectAccountJoinDao _projectAccountJoinDao; @Inject - private HostJoinDao hostJoinDao; + HostJoinDao hostJoinDao; @Inject - private VolumeJoinDao _volumeJoinDao; + VolumeJoinDao _volumeJoinDao; @Inject - private AccountDao _accountDao; + AccountDao _accountDao; @Inject - private AccountJoinDao _accountJoinDao; + AccountJoinDao _accountJoinDao; @Inject - private AsyncJobJoinDao _jobJoinDao; + AsyncJobJoinDao _jobJoinDao; @Inject - private StoragePoolJoinDao _poolJoinDao; + StoragePoolJoinDao _poolJoinDao; @Inject - private StoragePoolTagsDao _storageTagDao; + StoragePoolTagsDao _storageTagDao; @Inject - private HostTagsDao _hostTagDao; + HostTagsDao _hostTagDao; @Inject - private ImageStoreJoinDao _imageStoreJoinDao; + ImageStoreJoinDao _imageStoreJoinDao; @Inject - private DiskOfferingJoinDao _diskOfferingJoinDao; + DiskOfferingJoinDao _diskOfferingJoinDao; @Inject - private DiskOfferingDetailsDao _diskOfferingDetailsDao; + DiskOfferingDetailsDao _diskOfferingDetailsDao; @Inject - private ServiceOfferingJoinDao _srvOfferingJoinDao; + ServiceOfferingJoinDao _srvOfferingJoinDao; @Inject - private ServiceOfferingDao _srvOfferingDao; + ServiceOfferingDao _srvOfferingDao; @Inject - private ServiceOfferingDetailsDao _srvOfferingDetailsDao; + ServiceOfferingDetailsDao _srvOfferingDetailsDao; @Inject - private DiskOfferingDao _diskOfferingDao; + DiskOfferingDao _diskOfferingDao; @Inject - private DataCenterJoinDao _dcJoinDao; + DataCenterJoinDao _dcJoinDao; @Inject - private DomainRouterDao _routerDao; + DomainRouterDao _routerDao; @Inject - private HighAvailabilityManager _haMgr; + HighAvailabilityManager _haMgr; @Inject - private VMTemplateDao _templateDao; + VMTemplateDao _templateDao; @Inject - private TemplateJoinDao _templateJoinDao; + TemplateJoinDao _templateJoinDao; @Inject - private ResourceManager _resourceMgr; + ResourceManager _resourceMgr; @Inject - private ResourceMetaDataService _resourceMetaDataMgr; + ResourceMetaDataService _resourceMetaDataMgr; @Inject - private ResourceManagerUtil resourceManagerUtil; + ResourceManagerUtil resourceManagerUtil; @Inject - private AffinityGroupVMMapDao _affinityGroupVMMapDao; + AffinityGroupVMMapDao _affinityGroupVMMapDao; @Inject - private AffinityGroupJoinDao _affinityGroupJoinDao; + AffinityGroupJoinDao _affinityGroupJoinDao; @Inject - private DedicatedResourceDao _dedicatedDao; + DedicatedResourceDao _dedicatedDao; @Inject - private DomainManager _domainMgr; + DomainManager _domainMgr; @Inject - private AffinityGroupDomainMapDao _affinityGroupDomainMapDao; + AffinityGroupDomainMapDao _affinityGroupDomainMapDao; @Inject - private ResourceTagDao resourceTagDao; + ResourceTagDao resourceTagDao; @Inject - private DataStoreManager dataStoreManager; + DataStoreManager dataStoreManager; @Inject ManagementServerJoinDao managementServerJoinDao; @Inject - public VpcVirtualNetworkApplianceService routerService; + VpcVirtualNetworkApplianceService routerService; @Inject - private ResponseGenerator responseGenerator; + ResponseGenerator responseGenerator; @Inject - private RouterHealthCheckResultDao routerHealthCheckResultDao; + RouterHealthCheckResultDao routerHealthCheckResultDao; @Inject - private PrimaryDataStoreDao storagePoolDao; + PrimaryDataStoreDao storagePoolDao; @Inject - private StoragePoolDetailsDao _storagePoolDetailsDao; + StoragePoolDetailsDao _storagePoolDetailsDao; @Inject - private ProjectInvitationDao projectInvitationDao; + ProjectInvitationDao projectInvitationDao; @Inject - private TemplateDataStoreDao templateDataStoreDao; + TemplateDataStoreDao templateDataStoreDao; @Inject - private VMTemplatePoolDao templatePoolDao; + VMTemplatePoolDao templatePoolDao; @Inject - private SnapshotDataStoreDao snapshotDataStoreDao; + SnapshotDataStoreDao snapshotDataStoreDao; @Inject - private UserDao userDao; + UserDao userDao; @Inject - private VirtualMachineManager virtualMachineManager; + VirtualMachineManager virtualMachineManager; @Inject - private VolumeDao volumeDao; + VolumeDao volumeDao; @Inject - private ResourceIconDao resourceIconDao; + ResourceIconDao resourceIconDao; @Inject StorageManager storageManager; @Inject - private ManagementServerHostDao msHostDao; + ManagementServerHostDao msHostDao; @Inject - private SecondaryStorageHeuristicDao secondaryStorageHeuristicDao; + SecondaryStorageHeuristicDao secondaryStorageHeuristicDao; @Inject - private NetworkDao networkDao; + NetworkDao networkDao; @Inject - private IPAddressDao ipAddressDao; + IPAddressDao ipAddressDao; @Inject - private NicDao nicDao; + NicDao nicDao; @Inject - private HostDao hostDao; + HostDao hostDao; @Inject - private OutOfBandManagementDao outOfBandManagementDao; + OutOfBandManagementDao outOfBandManagementDao; @Inject - private InstanceGroupVMMapDao instanceGroupVMMapDao; + InstanceGroupVMMapDao instanceGroupVMMapDao; @Inject - private AffinityGroupVMMapDao affinityGroupVMMapDao; + AffinityGroupVMMapDao affinityGroupVMMapDao; @Inject - private UserVmDetailsDao userVmDetailsDao; + UserVmDetailsDao userVmDetailsDao; @Inject - private SSHKeyPairDao sshKeyPairDao; + SSHKeyPairDao sshKeyPairDao; @Inject - private BackupOfferingDao backupOfferingDao; + BackupOfferingDao backupOfferingDao; @Inject - private AutoScaleVmGroupDao autoScaleVmGroupDao; + AutoScaleVmGroupDao autoScaleVmGroupDao; @Inject - private AutoScaleVmGroupVmMapDao autoScaleVmGroupVmMapDao; + AutoScaleVmGroupVmMapDao autoScaleVmGroupVmMapDao; @Inject - private SnapshotJoinDao snapshotJoinDao; + SnapshotJoinDao snapshotJoinDao; @Inject - private ObjectStoreDao objectStoreDao; + ObjectStoreDao objectStoreDao; @Inject - private BucketDao bucketDao; + BucketDao bucketDao; @Inject EntityManager entityManager; @Inject - private PublicIpQuarantineDao publicIpQuarantineDao; + PublicIpQuarantineDao publicIpQuarantineDao; @Inject - private StoragePoolHostDao storagePoolHostDao; + StoragePoolHostDao storagePoolHostDao; @Inject - private ClusterDao clusterDao; + ClusterDao clusterDao; @Inject - private ManagementServerHostPeerJoinDao mshostPeerJoinDao; + ManagementServerHostPeerJoinDao mshostPeerJoinDao; @Inject private AsyncJobManager jobManager; @@ -666,7 +666,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q @Override public ListResponse searchForUsers(ResponseView responseView, ListUsersCmd cmd) throws PermissionDeniedException { Pair, Integer> result = searchForUsersInternal(cmd); - ListResponse response = new ListResponse(); + ListResponse response = new ListResponse<>(); if (CallContext.current().getCallingAccount().getType() == Account.Type.ADMIN) { responseView = ResponseView.Full; } @@ -842,7 +842,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q @Override public ListResponse searchForEvents(ListEventsCmd cmd) { Pair, Integer> result = searchForEventsInternal(cmd); - ListResponse response = new ListResponse(); + ListResponse response = new ListResponse<>(); List eventResponses = ViewResponseHelper.createEventResponse(result.first().toArray(new EventJoinVO[result.first().size()])); response.setResponses(eventResponses, result.second()); return response; @@ -914,7 +914,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q } } - Ternary domainIdRecursiveListProject = new Ternary(cmd.getDomainId(), cmd.isRecursive(), null); + Ternary domainIdRecursiveListProject = new Ternary<>(cmd.getDomainId(), cmd.isRecursive(), null); accountMgr.buildACLSearchParameters(caller, id, cmd.getAccountName(), cmd.getProjectId(), permittedAccounts, domainIdRecursiveListProject, cmd.listAll(), false); Long domainId = domainIdRecursiveListProject.first(); Boolean isRecursive = domainIdRecursiveListProject.second(); @@ -1005,7 +1005,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q sc.setParameters("archived", cmd.getArchived()); } - Pair, Integer> eventPair = null; + Pair, Integer> eventPair; // event_view will not have duplicate rows for each event, so // searchAndCount should be good enough. if ((entryTime != null) && (duration != null)) { @@ -1042,7 +1042,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q @Override public ListResponse listTags(ListTagsCmd cmd) { Pair, Integer> tags = listTagsInternal(cmd); - ListResponse response = new ListResponse(); + ListResponse response = new ListResponse<>(); List tagResponses = ViewResponseHelper.createResourceTagResponse(false, tags.first().toArray(new ResourceTagJoinVO[tags.first().size()])); response.setResponses(tagResponses, tags.second()); return response; @@ -1050,7 +1050,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q private Pair, Integer> listTagsInternal(ListTagsCmd cmd) { Account caller = CallContext.current().getCallingAccount(); - List permittedAccounts = new ArrayList(); + List permittedAccounts = new ArrayList<>(); String key = cmd.getKey(); String value = cmd.getValue(); String resourceId = cmd.getResourceId(); @@ -1070,7 +1070,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q } } - Ternary domainIdRecursiveListProject = new Ternary(cmd.getDomainId(), cmd.isRecursive(), null); + Ternary domainIdRecursiveListProject = new Ternary<>(cmd.getDomainId(), cmd.isRecursive(), null); accountMgr.buildACLSearchParameters(caller, null, cmd.getAccountName(), projectId, permittedAccounts, domainIdRecursiveListProject, listAll, false); Long domainId = domainIdRecursiveListProject.first(); @@ -1122,14 +1122,13 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q sc.setParameters("customer", customerName); } - Pair, Integer> result = _resourceTagJoinDao.searchAndCount(sc, searchFilter); - return result; + return _resourceTagJoinDao.searchAndCount(sc, searchFilter); } @Override public ListResponse searchForVmGroups(ListVMGroupsCmd cmd) { Pair, Integer> groups = searchForVmGroupsInternal(cmd); - ListResponse response = new ListResponse(); + ListResponse response = new ListResponse<>(); List grpResponses = ViewResponseHelper.createInstanceGroupResponse(groups.first().toArray(new InstanceGroupJoinVO[groups.first().size()])); response.setResponses(grpResponses, groups.second()); return response; @@ -1141,9 +1140,9 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q String keyword = cmd.getKeyword(); Account caller = CallContext.current().getCallingAccount(); - List permittedAccounts = new ArrayList(); + List permittedAccounts = new ArrayList<>(); - Ternary domainIdRecursiveListProject = new Ternary(cmd.getDomainId(), cmd.isRecursive(), null); + Ternary domainIdRecursiveListProject = new Ternary<>(cmd.getDomainId(), cmd.isRecursive(), null); accountMgr.buildACLSearchParameters(caller, id, cmd.getAccountName(), cmd.getProjectId(), permittedAccounts, domainIdRecursiveListProject, cmd.listAll(), false); Long domainId = domainIdRecursiveListProject.first(); Boolean isRecursive = domainIdRecursiveListProject.second(); @@ -1337,7 +1336,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q Filter searchFilter = new Filter(UserVmVO.class, "id", true, cmd.getStartIndex(), cmd.getPageSizeVal()); - List ids = null; + List ids; if (cmd.getId() != null) { if (cmd.getIds() != null && !cmd.getIds().isEmpty()) { throw new InvalidParameterValueException("Specify either id or ids but not both parameters"); @@ -1703,7 +1702,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q @Override public ListResponse searchForSecurityGroups(ListSecurityGroupsCmd cmd) { Pair, Integer> result = searchForSecurityGroupsInternal(cmd); - ListResponse response = new ListResponse(); + ListResponse response = new ListResponse<>(); List routerResponses = ViewResponseHelper.createSecurityGroupResponses(result.first()); response.setResponses(routerResponses, result.second()); return response; @@ -1715,7 +1714,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q String securityGroup = cmd.getSecurityGroupName(); Long id = cmd.getId(); Object keyword = cmd.getKeyword(); - List permittedAccounts = new ArrayList(); + List permittedAccounts = new ArrayList<>(); Map tags = cmd.getTags(); if (instanceId != null) { @@ -1727,7 +1726,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q return listSecurityGroupRulesByVM(instanceId.longValue(), cmd.getStartIndex(), cmd.getPageSizeVal()); } - Ternary domainIdRecursiveListProject = new Ternary(cmd.getDomainId(), cmd.isRecursive(), null); + Ternary domainIdRecursiveListProject = new Ternary<>(cmd.getDomainId(), cmd.isRecursive(), null); accountMgr.buildACLSearchParameters(caller, id, cmd.getAccountName(), cmd.getProjectId(), permittedAccounts, domainIdRecursiveListProject, cmd.listAll(), false); Long domainId = domainIdRecursiveListProject.first(); Boolean isRecursive = domainIdRecursiveListProject.second(); @@ -1786,7 +1785,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q sgIds[i++] = v.getId(); } List sgs = _securityGroupJoinDao.searchByIds(sgIds); - return new Pair, Integer>(sgs, count); + return new Pair<>(sgs, count); } private Pair, Integer> listSecurityGroupRulesByVM(long vmId, long pageInd, long pageSize) { @@ -1795,7 +1794,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q Integer count = sgVmMappingPair.second(); if (count.intValue() == 0) { // handle empty result cases - return new Pair, Integer>(new ArrayList(), count); + return new Pair<>(new ArrayList<>(), count); } List sgVmMappings = sgVmMappingPair.first(); Long[] sgIds = new Long[sgVmMappings.size()]; @@ -1804,14 +1803,14 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q sgIds[i++] = sgVm.getSecurityGroupId(); } List sgs = _securityGroupJoinDao.searchByIds(sgIds); - return new Pair, Integer>(sgs, count); + return new Pair<>(sgs, count); } @Override public ListResponse searchForRouters(ListRoutersCmd cmd) { Pair, Integer> result = searchForRoutersInternal(cmd, cmd.getId(), cmd.getRouterName(), cmd.getState(), cmd.getZoneId(), cmd.getPodId(), cmd.getClusterId(), cmd.getHostId(), cmd.getKeyword(), cmd.getNetworkId(), cmd.getVpcId(), cmd.getForVpc(), cmd.getRole(), cmd.getVersion(), cmd.isHealthCheckFailed()); - ListResponse response = new ListResponse(); + ListResponse response = new ListResponse<>(); List routerResponses = ViewResponseHelper.createDomainRouterResponse(result.first().toArray(new DomainRouterJoinVO[result.first().size()])); if (VirtualNetworkApplianceManager.RouterHealthChecksEnabled.value()) { for (DomainRouterResponse res : routerResponses) { @@ -1831,7 +1830,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q public ListResponse searchForInternalLbVms(ListInternalLBVMsCmd cmd) { Pair, Integer> result = searchForRoutersInternal(cmd, cmd.getId(), cmd.getRouterName(), cmd.getState(), cmd.getZoneId(), cmd.getPodId(), null, cmd.getHostId(), cmd.getKeyword(), cmd.getNetworkId(), cmd.getVpcId(), cmd.getForVpc(), cmd.getRole(), null, null); - ListResponse response = new ListResponse(); + ListResponse response = new ListResponse<>(); List routerResponses = ViewResponseHelper.createDomainRouterResponse(result.first().toArray(new DomainRouterJoinVO[result.first().size()])); if (VirtualNetworkApplianceManager.RouterHealthChecksEnabled.value()) { for (DomainRouterResponse res : routerResponses) { @@ -1852,9 +1851,9 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q Long hostId, String keyword, Long networkId, Long vpcId, Boolean forVpc, String role, String version, Boolean isHealthCheckFailed) { Account caller = CallContext.current().getCallingAccount(); - List permittedAccounts = new ArrayList(); + List permittedAccounts = new ArrayList<>(); - Ternary domainIdRecursiveListProject = new Ternary(cmd.getDomainId(), cmd.isRecursive(), null); + Ternary domainIdRecursiveListProject = new Ternary<>(cmd.getDomainId(), cmd.isRecursive(), null); accountMgr.buildACLSearchParameters(caller, id, cmd.getAccountName(), cmd.getProjectId(), permittedAccounts, domainIdRecursiveListProject, cmd.listAll(), false); Long domainId = domainIdRecursiveListProject.first(); Boolean isRecursive = domainIdRecursiveListProject.second(); @@ -1910,7 +1909,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q sb.and("routerId", sb.entity().getId(), SearchCriteria.Op.NIN); } } else if (isHealthCheckFailed) { - return new Pair, Integer>(Collections.emptyList(), 0); + return new Pair<>(Collections.emptyList(), 0); } } @@ -1990,13 +1989,13 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q vrIds[i++] = v.getId(); } List vrs = _routerJoinDao.searchByIds(vrIds); - return new Pair, Integer>(vrs, count); + return new Pair<>(vrs, count); } @Override public ListResponse listProjects(ListProjectsCmd cmd) { Pair, Integer> projects = listProjectsInternal(cmd); - ListResponse response = new ListResponse(); + ListResponse response = new ListResponse<>(); List projectResponses = ViewResponseHelper.createProjectResponse(cmd.getDetails(), projects.first().toArray(new ProjectJoinVO[projects.first().size()])); response.setResponses(projectResponses, projects.second()); return response; @@ -2164,13 +2163,13 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q prjIds[i++] = v.getId(); } List prjs = _projectJoinDao.searchByIds(prjIds); - return new Pair, Integer>(prjs, count); + return new Pair<>(prjs, count); } @Override public ListResponse listProjectInvitations(ListProjectInvitationsCmd cmd) { Pair, Integer> invites = listProjectInvitationsInternal(cmd); - ListResponse response = new ListResponse(); + ListResponse response = new ListResponse<>(); List projectInvitationResponses = ViewResponseHelper.createProjectInvitationResponse(invites.first().toArray(new ProjectInvitationJoinVO[invites.first().size()])); response.setResponses(projectInvitationResponses, invites.second()); @@ -2192,9 +2191,9 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q Account caller = CallContext.current().getCallingAccount(); User callingUser = CallContext.current().getCallingUser(); - List permittedAccounts = new ArrayList(); + List permittedAccounts = new ArrayList<>(); - Ternary domainIdRecursiveListProject = new Ternary(domainId, isRecursive, null); + Ternary domainIdRecursiveListProject = new Ternary<>(domainId, isRecursive, null); accountMgr.buildACLSearchParameters(caller, id, accountName, projectId, permittedAccounts, domainIdRecursiveListProject, listAll, true); domainId = domainIdRecursiveListProject.first(); isRecursive = domainIdRecursiveListProject.second(); @@ -2246,7 +2245,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q @Override public ListResponse listProjectAccounts(ListProjectAccountsCmd cmd) { Pair, Integer> projectAccounts = listProjectAccountsInternal(cmd); - ListResponse response = new ListResponse(); + ListResponse response = new ListResponse<>(); List projectResponses = ViewResponseHelper.createProjectAccountResponse(projectAccounts.first().toArray(new ProjectAccountJoinVO[projectAccounts.first().size()])); response.setResponses(projectResponses, projectAccounts.second()); return response; @@ -2320,7 +2319,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q // Right now it is handled separately outside this QueryService logger.debug(">>>Searching for hosts>>>"); Pair, Integer> hosts = searchForServersInternal(cmd); - ListResponse response = new ListResponse(); + ListResponse response = new ListResponse<>(); logger.debug(">>>Generating Response>>>"); List hostResponses = ViewResponseHelper.createHostResponse(cmd.getDetails(), hosts.first().toArray(new HostJoinVO[hosts.first().size()])); response.setResponses(hostResponses, hosts.second()); @@ -2741,7 +2740,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q @Override public ListResponse searchForDomains(ListDomainsCmd cmd) { Pair, Integer> result = searchForDomainsInternal(cmd); - ListResponse response = new ListResponse(); + ListResponse response = new ListResponse<>(); ResponseView respView = ResponseView.Restricted; if (cmd instanceof ListDomainsCmdByAdmin) { @@ -2843,7 +2842,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q @Override public ListResponse searchForAccounts(ListAccountsCmd cmd) { Pair, Integer> result = searchForAccountsInternal(cmd); - ListResponse response = new ListResponse(); + ListResponse response = new ListResponse<>(); ResponseView respView = ResponseView.Restricted; if (cmd instanceof ListAccountsCmdByAdmin) { @@ -3029,7 +3028,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q @Override public ListResponse searchForAsyncJobs(ListAsyncJobsCmd cmd) { Pair, Integer> result = searchForAsyncJobsInternal(cmd); - ListResponse response = new ListResponse(); + ListResponse response = new ListResponse<>(); List jobResponses = ViewResponseHelper.createAsyncJobResponse(result.first().toArray(new AsyncJobJoinVO[result.first().size()])); response.setResponses(jobResponses, result.second()); return response; @@ -3039,9 +3038,9 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q Account caller = CallContext.current().getCallingAccount(); - List permittedAccounts = new ArrayList(); + List permittedAccounts = new ArrayList<>(); - Ternary domainIdRecursiveListProject = new Ternary(cmd.getDomainId(), cmd.isRecursive(), null); + Ternary domainIdRecursiveListProject = new Ternary<>(cmd.getDomainId(), cmd.isRecursive(), null); accountMgr.buildACLSearchParameters(caller, null, cmd.getAccountName(), null, permittedAccounts, domainIdRecursiveListProject, cmd.listAll(), false); Long domainId = domainIdRecursiveListProject.first(); Boolean isRecursive = domainIdRecursiveListProject.second(); @@ -3201,7 +3200,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q @Override public ListResponse searchForStorageTags(ListStorageTagsCmd cmd) { Pair, Integer> result = searchForStorageTagsInternal(cmd); - ListResponse response = new ListResponse(); + ListResponse response = new ListResponse<>(); List tagResponses = ViewResponseHelper.createStorageTagResponse(result.first().toArray(new StoragePoolTagVO[result.first().size()])); response.setResponses(tagResponses, result.second()); @@ -3236,13 +3235,13 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q List vrs = _storageTagDao.searchByIds(vrIds); - return new Pair, Integer>(vrs, count); + return new Pair<>(vrs, count); } @Override public ListResponse searchForHostTags(ListHostTagsCmd cmd) { Pair, Integer> result = searchForHostTagsInternal(cmd); - ListResponse response = new ListResponse(); + ListResponse response = new ListResponse<>(); List tagResponses = ViewResponseHelper.createHostTagResponse(result.first().toArray(new HostTagVO[result.first().size()])); response.setResponses(tagResponses, result.second()); @@ -3277,7 +3276,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q List vrs = _hostTagDao.searchByIds(vrIds); - return new Pair, Integer>(vrs, count); + return new Pair<>(vrs, count); } @Override @@ -3360,14 +3359,14 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q vrIds[i++] = v.getId(); } List vrs = _imageStoreJoinDao.searchByIds(vrIds); - return new Pair, Integer>(vrs, count); + return new Pair<>(vrs, count); } @Override public ListResponse searchForSecondaryStagingStores(ListSecondaryStagingStoresCmd cmd) { Pair, Integer> result = searchForCacheStoresInternal(cmd); - ListResponse response = new ListResponse(); + ListResponse response = new ListResponse<>(); List poolResponses = ViewResponseHelper.createImageStoreResponse(result.first().toArray(new ImageStoreJoinVO[result.first().size()])); response.setResponses(poolResponses, result.second()); @@ -3439,7 +3438,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q vrIds[i++] = v.getId(); } List vrs = _imageStoreJoinDao.searchByIds(vrIds); - return new Pair, Integer>(vrs, count); + return new Pair<>(vrs, count); } @@ -4263,7 +4262,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q @Override public ListResponse listDataCenters(ListZonesCmd cmd) { Pair, Integer> result = listDataCentersInternal(cmd); - ListResponse response = new ListResponse(); + ListResponse response = new ListResponse<>(); ResponseView respView = ResponseView.Restricted; if (cmd instanceof ListZonesCmdByAdmin || CallContext.current().getCallingAccount().getType() == Account.Type.ADMIN) { @@ -4350,7 +4349,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q // list // find all domain Id up to root domain for this account - List domainIds = new ArrayList(); + List domainIds = new ArrayList<>(); DomainVO domainRecord = _domainDao.findById(account.getDomainId()); if (domainRecord == null) { logger.error("Could not find the domainId for account: {}", account); @@ -4390,7 +4389,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q // it was decided to return all zones for the domain admin, and // everything above till root, as well as zones till the domain // leaf - List domainIds = new ArrayList(); + List domainIds = new ArrayList<>(); DomainVO domainRecord = _domainDao.findById(account.getDomainId()); if (domainRecord == null) { logger.error("Could not find the domainId for account: {}", account); @@ -4430,8 +4429,8 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q // one VM running there Boolean available = cmd.isAvailable(); if (account != null) { - if ((available != null) && Boolean.FALSE.equals(available)) { - Set dcIds = new HashSet(); // data centers with + if (Boolean.FALSE.equals(available)) { + Set dcIds = new HashSet<>(); // data centers with // at least one VM // running List routers = _routerDao.listBy(account.getId()); @@ -4439,7 +4438,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q dcIds.add(router.getDataCenterId()); } if (dcIds.size() == 0) { - return new Pair, Integer>(new ArrayList(), 0); + return new Pair<>(new ArrayList<>(), 0); } else { sc.addAnd("id", SearchCriteria.Op.IN, dcIds.toArray()); } @@ -4463,7 +4462,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q private List removeDedicatedZoneNotSuitabe(List domainIds) { // remove dedicated zone of other domain - List dedicatedZoneIds = new ArrayList(); + List dedicatedZoneIds = new ArrayList<>(); List dedicatedResources = _dedicatedDao.listZonesNotInDomainIds(domainIds); for (DedicatedResourceVO dr : dedicatedResources) { if (dr != null) { @@ -4504,7 +4503,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q @Override public ListResponse listTemplates(ListTemplatesCmd cmd) { Pair, Integer> result = searchForTemplatesInternal(cmd); - ListResponse response = new ListResponse(); + ListResponse response = new ListResponse<>(); ResponseView respView = ResponseView.Restricted; if (cmd instanceof ListTemplatesCmdByAdmin) { @@ -4532,14 +4531,14 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q listAll = true; } - List permittedAccountIds = new ArrayList(); + List permittedAccountIds = new ArrayList<>(); Ternary domainIdRecursiveListProject = new Ternary<>(cmd.getDomainId(), cmd.isRecursive(), null); accountMgr.buildACLSearchParameters( caller, id, cmd.getAccountName(), cmd.getProjectId(), permittedAccountIds, domainIdRecursiveListProject, listAll, false ); ListProjectResourcesCriteria listProjectResourcesCriteria = domainIdRecursiveListProject.third(); - List permittedAccounts = new ArrayList(); + List permittedAccounts = new ArrayList<>(); for (Long accountId : permittedAccountIds) { permittedAccounts.add(accountMgr.getAccount(accountId)); } @@ -4578,11 +4577,11 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q if (!isIso) { hypers = _resourceMgr.listAvailHypervisorInZone(null); if (hypers == null || hypers.isEmpty()) { - return new Pair, Integer>(new ArrayList(), 0); + return new Pair<>(new ArrayList<>(), 0); } } - VMTemplateVO template = null; + VMTemplateVO template; Filter searchFilter = new Filter(TemplateJoinVO.class, "sortKey", SortKeyAscending.value(), startIndex, pageSize); searchFilter.addOrderBy(TemplateJoinVO.class, "tempZonePair", SortKeyAscending.value()); @@ -4653,7 +4652,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q sc.addAnd("id", SearchCriteria.Op.EQ, templateId); } else { - DomainVO domain = null; + DomainVO domain; if (!permittedAccounts.isEmpty()) { domain = _domainDao.findById(permittedAccounts.get(0).getDomainId()); } else { @@ -4675,15 +4674,15 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q sc.addAnd("domainPath", SearchCriteria.Op.LIKE, domain.getPath() + "%"); } - List relatedDomainIds = new ArrayList(); - List permittedAccountIds = new ArrayList(); + List relatedDomainIds = new ArrayList<>(); + List permittedAccountIds = new ArrayList<>(); if (!permittedAccounts.isEmpty()) { for (Account account : permittedAccounts) { permittedAccountIds.add(account.getId()); boolean publicTemplates = (templateFilter == TemplateFilter.featured || templateFilter == TemplateFilter.community); // get all parent domain ID's all the way till root domain - DomainVO domainTreeNode = null; + DomainVO domainTreeNode; //if template filter is featured, or community, all child domains should be included in search if (publicTemplates) { domainTreeNode = _domainDao.findById(Domain.ROOT_DOMAIN); @@ -4918,7 +4917,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q // sc.addAnd("removed", SearchCriteria.Op.NULL); // search unique templates and find details by Ids - Pair, Integer> uniqueTmplPair = null; + Pair, Integer> uniqueTmplPair; if (showRemovedTmpl) { uniqueTmplPair = _templateJoinDao.searchIncludingRemovedAndCount(sc, searchFilter); } else { @@ -4949,7 +4948,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q return templateDataPair; } List templateData = templateDataPair.first(); - List templates = null; + List templates; if (showUnique) { Long[] templateIds = templateData.stream().map(template -> template.getId()).toArray(Long[]::new); templates = _templateJoinDao.findByDistinctIds(templateIds); @@ -4958,7 +4957,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q templates = _templateJoinDao.searchByTemplateZonePair(showRemoved, templateZonePairs); } - return new Pair, Integer>(templates, count); + return new Pair<>(templates, count); } @Override @@ -5079,7 +5078,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q if (HypervisorType.KVM.equals(hypervisorType)) { options.put(VmDetailConstants.CPU_THREAD_PER_CORE, Collections.emptyList()); options.put(VmDetailConstants.NIC_ADAPTER, Arrays.asList("e1000", "virtio", "rtl8139", "vmxnet3", "ne2k_pci")); - options.put(VmDetailConstants.ROOT_DISK_CONTROLLER, Arrays.asList("osdefault", "ide", "scsi", "virtio")); + options.put(VmDetailConstants.ROOT_DISK_CONTROLLER, Arrays.asList("osdefault", "ide", "scsi", "virtio", "virtio-blk")); options.put(VmDetailConstants.VIDEO_HARDWARE, Arrays.asList("cirrus", "vga", "qxl", "virtio")); options.put(VmDetailConstants.VIDEO_RAM, Collections.emptyList()); options.put(VmDetailConstants.IO_POLICY, Arrays.asList("threads", "native", "io_uring", "storage_specific")); @@ -5133,8 +5132,8 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q return listAffinityGroupsByVM(vmId.longValue(), startIndex, pageSize); } - List permittedAccounts = new ArrayList(); - Ternary ternary = new Ternary(domainId, isRecursive, null); + List permittedAccounts = new ArrayList<>(); + Ternary ternary = new Ternary<>(domainId, isRecursive, null); accountMgr.buildACLSearchParameters(caller, affinityGroupId, accountName, projectId, permittedAccounts, ternary, listAll, false); @@ -5150,7 +5149,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q Pair, Integer> uniqueGroupsPair = _affinityGroupJoinDao.searchAndCount(sc, searchFilter); // search group details by ids - List affinityGroups = new ArrayList(); + List affinityGroups = new ArrayList<>(); Integer count = uniqueGroupsPair.second(); if (count.intValue() != 0) { @@ -5166,7 +5165,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q if (!permittedAccounts.isEmpty()) { // add domain level affinity groups if (domainId != null) { - SearchCriteria scDomain = buildAffinityGroupSearchCriteria(null, isRecursive, new ArrayList(), listProjectResourcesCriteria, affinityGroupId, + SearchCriteria scDomain = buildAffinityGroupSearchCriteria(null, isRecursive, new ArrayList<>(), listProjectResourcesCriteria, affinityGroupId, affinityGroupName, affinityGroupType, keyword); Pair, Integer> groupsPair = listDomainLevelAffinityGroups(scDomain, searchFilter, domainId); affinityGroups.addAll(groupsPair.first()); @@ -5175,7 +5174,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q for (Long permAcctId : permittedAccounts) { Account permittedAcct = _accountDao.findById(permAcctId); - SearchCriteria scDomain = buildAffinityGroupSearchCriteria(null, isRecursive, new ArrayList(), listProjectResourcesCriteria, affinityGroupId, + SearchCriteria scDomain = buildAffinityGroupSearchCriteria(null, isRecursive, new ArrayList<>(), listProjectResourcesCriteria, affinityGroupId, affinityGroupName, affinityGroupType, keyword); Pair, Integer> groupsPair = listDomainLevelAffinityGroups(scDomain, searchFilter, permittedAcct.getDomainId()); affinityGroups.addAll(groupsPair.first()); @@ -5184,14 +5183,14 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q } } else if (((permittedAccounts.isEmpty()) && (domainId != null) && isRecursive)) { // list all domain level affinity groups for the domain admin case - SearchCriteria scDomain = buildAffinityGroupSearchCriteria(null, isRecursive, new ArrayList(), listProjectResourcesCriteria, affinityGroupId, affinityGroupName, + SearchCriteria scDomain = buildAffinityGroupSearchCriteria(null, isRecursive, new ArrayList<>(), listProjectResourcesCriteria, affinityGroupId, affinityGroupName, affinityGroupType, keyword); Pair, Integer> groupsPair = listDomainLevelAffinityGroups(scDomain, searchFilter, domainId); affinityGroups.addAll(groupsPair.first()); count += groupsPair.second(); } - return new Pair, Integer>(affinityGroups, count); + return new Pair<>(affinityGroups, count); } @@ -5277,7 +5276,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q Integer count = agVmMappingPair.second(); if (count.intValue() == 0) { // handle empty result cases - return new Pair, Integer>(new ArrayList(), count); + return new Pair<>(new ArrayList<>(), count); } List agVmMappings = agVmMappingPair.first(); Long[] agIds = new Long[agVmMappings.size()]; @@ -5286,11 +5285,11 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q agIds[i++] = agVm.getAffinityGroupId(); } List ags = _affinityGroupJoinDao.searchByIds(agIds); - return new Pair, Integer>(ags, count); + return new Pair<>(ags, count); } private Pair, Integer> listDomainLevelAffinityGroups(SearchCriteria sc, Filter searchFilter, long domainId) { - List affinityGroupIds = new ArrayList(); + List affinityGroupIds = new ArrayList<>(); Set allowedDomains = _domainMgr.getDomainParentIds(domainId); List maps = _affinityGroupDomainMapDao.listByDomain(allowedDomains.toArray()); @@ -5313,7 +5312,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q Integer count = uniqueGroupsPair.second(); if (count.intValue() == 0) { // empty result - return new Pair<>(new ArrayList(), 0); + return new Pair<>(new ArrayList<>(), 0); } List uniqueGroups = uniqueGroupsPair.first(); Long[] vrIds = new Long[uniqueGroups.size()]; @@ -5324,7 +5323,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q List vrs = _affinityGroupJoinDao.searchByIds(vrIds); return new Pair<>(vrs, count); } else { - return new Pair<>(new ArrayList(), 0); + return new Pair<>(new ArrayList<>(), 0); } } @@ -5355,7 +5354,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q } } - List detailList = new ArrayList(); + List detailList = new ArrayList<>(); ResourceDetail requestedDetail = null; if (key == null) { @@ -5369,7 +5368,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q detailList = _resourceMetaDataMgr.getDetails(resourceType, key, value, forDisplay); } - List responseList = new ArrayList(); + List responseList = new ArrayList<>(); if (requestedDetail != null) { ResourceDetailResponse detailResponse = createResourceDetailsResponse(requestedDetail, resourceType); responseList.add(detailResponse); @@ -5436,7 +5435,6 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q mgmtResponse.setLastServerStart(mgmt.getLastJvmStart()); mgmtResponse.setLastServerStop(mgmt.getLastJvmStop()); mgmtResponse.setLastBoot(mgmt.getLastSystemBoot()); - mgmtResponse.setServiceIp(mgmt.getServiceIP()); if (listPeers) { List peers = mshostPeerJoinDao.listByOwnerMshostId(mgmt.getId()); for (ManagementServerHostPeerJoinVO peer: peers) { @@ -5445,6 +5443,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q } mgmtResponse.setAgentsCount((long) hostDao.countByMs(mgmt.getMsid())); mgmtResponse.setPendingJobsCount(jobManager.countPendingNonPseudoJobs(mgmt.getMsid())); + mgmtResponse.setIpAddress(mgmt.getServiceIP()); mgmtResponse.setObjectName("managementserver"); return mgmtResponse; } @@ -5621,7 +5620,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q Filter searchFilter = new Filter(SnapshotJoinVO.class, "snapshotStorePair", SortKeyAscending.value(), startIndex, pageSize); List permittedAccountIds = new ArrayList<>(); - Ternary domainIdRecursiveListProject = new Ternary(domainId, isRecursive, null); + Ternary domainIdRecursiveListProject = new Ternary<>(domainId, isRecursive, null); accountMgr.buildACLSearchParameters(caller, id, accountName, projectId, permittedAccountIds, domainIdRecursiveListProject, listAll, false); ListProjectResourcesCriteria listProjectResourcesCriteria = domainIdRecursiveListProject.third(); domainId = domainIdRecursiveListProject.first(); @@ -5754,7 +5753,6 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q Integer count = snapshotDataPair.second(); if (count == 0) { - // empty result return snapshotDataPair; } List snapshotData = snapshotDataPair.first(); @@ -5764,13 +5762,12 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q } else { snapshots = snapshotJoinDao.searchBySnapshotStorePair(snapshotData.stream().map(SnapshotJoinVO::getSnapshotStorePair).toArray(String[]::new)); } - return new Pair<>(snapshots, count); } public ListResponse searchForObjectStores(ListObjectStoragePoolsCmd cmd) { Pair, Integer> result = searchForObjectStoresInternal(cmd); - ListResponse response = new ListResponse(); + ListResponse response = new ListResponse<>(); List poolResponses = ViewResponseHelper.createObjectStoreResponse(result.first().toArray(new ObjectStoreVO[result.first().size()])); response.setResponses(poolResponses, result.second()); @@ -5855,7 +5852,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q Long startIndex = cmd.getStartIndex(); Long pageSize = cmd.getPageSizeVal(); Account caller = CallContext.current().getCallingAccount(); - List permittedAccounts = new ArrayList(); + List permittedAccounts = new ArrayList<>(); // Verify parameters if (id != null) { @@ -5867,7 +5864,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q List ids = getIdsListFromCmd(cmd.getId(), cmd.getIds()); - Ternary domainIdRecursiveListProject = new Ternary(cmd.getDomainId(), + Ternary domainIdRecursiveListProject = new Ternary<>(cmd.getDomainId(), cmd.isRecursive(), null); accountMgr.buildACLSearchParameters(caller, id, cmd.getAccountName(), cmd.getProjectId(), permittedAccounts, domainIdRecursiveListProject, cmd.listAll(), false); Long domainId = domainIdRecursiveListProject.first(); diff --git a/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java b/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java index 7d5658f6782..bf5c4666984 100644 --- a/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java +++ b/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java @@ -453,6 +453,10 @@ public class ViewResponseHelper { resourceLimitMap.put(Resource.ResourceType.primary_storage, domainJoinVO.getPrimaryStorageLimit()); resourceLimitMap.put(Resource.ResourceType.secondary_storage, domainJoinVO.getSecondaryStorageLimit()); resourceLimitMap.put(Resource.ResourceType.project, domainJoinVO.getProjectLimit()); + resourceLimitMap.put(Resource.ResourceType.backup, domainJoinVO.getBackupLimit()); + resourceLimitMap.put(Resource.ResourceType.backup_storage, domainJoinVO.getBackupStorageLimit()); + resourceLimitMap.put(Resource.ResourceType.bucket, domainJoinVO.getBucketLimit()); + resourceLimitMap.put(Resource.ResourceType.object_storage, domainJoinVO.getObjectStorageLimit()); } private static void copyResourceLimitsFromMap(Map resourceLimitMap, DomainJoinVO domainJoinVO){ @@ -468,6 +472,10 @@ public class ViewResponseHelper { domainJoinVO.setPrimaryStorageLimit(resourceLimitMap.get(Resource.ResourceType.primary_storage)); domainJoinVO.setSecondaryStorageLimit(resourceLimitMap.get(Resource.ResourceType.secondary_storage)); domainJoinVO.setProjectLimit(resourceLimitMap.get(Resource.ResourceType.project)); + domainJoinVO.setBackupLimit(resourceLimitMap.get(Resource.ResourceType.backup)); + domainJoinVO.setBackupStorageLimit(resourceLimitMap.get(Resource.ResourceType.backup_storage)); + domainJoinVO.setBucketLimit(resourceLimitMap.get(Resource.ResourceType.bucket)); + domainJoinVO.setObjectStorageLimit(resourceLimitMap.get(Resource.ResourceType.object_storage)); } private static void setParentResourceLimitIfNeeded(Map resourceLimitMap, DomainJoinVO domainJoinVO, List domainsCopy) { @@ -486,6 +494,10 @@ public class ViewResponseHelper { Long primaryStorageLimit = resourceLimitMap.get(Resource.ResourceType.primary_storage); Long secondaryStorageLimit = resourceLimitMap.get(Resource.ResourceType.secondary_storage); Long projectLimit = resourceLimitMap.get(Resource.ResourceType.project); + Long backupLimit = resourceLimitMap.get(Resource.ResourceType.backup); + Long backupStorageLimit = resourceLimitMap.get(Resource.ResourceType.backup_storage); + Long bucketLimit = resourceLimitMap.get(Resource.ResourceType.bucket); + Long objectStorageLimit = resourceLimitMap.get(Resource.ResourceType.object_storage); if (vmLimit == null) { vmLimit = parentDomainJoinVO.getVmLimit(); @@ -535,6 +547,22 @@ public class ViewResponseHelper { projectLimit = parentDomainJoinVO.getProjectLimit(); resourceLimitMap.put(Resource.ResourceType.project, projectLimit); } + if (backupLimit == null) { + backupLimit = parentDomainJoinVO.getBackupLimit(); + resourceLimitMap.put(Resource.ResourceType.backup, backupLimit); + } + if (backupStorageLimit == null) { + backupStorageLimit = parentDomainJoinVO.getBackupStorageLimit(); + resourceLimitMap.put(Resource.ResourceType.backup_storage, backupStorageLimit); + } + if (bucketLimit == null) { + bucketLimit = parentDomainJoinVO.getBucketLimit(); + resourceLimitMap.put(Resource.ResourceType.bucket, bucketLimit); + } + if (objectStorageLimit == null) { + objectStorageLimit = parentDomainJoinVO.getObjectStorageLimit(); + resourceLimitMap.put(Resource.ResourceType.object_storage, objectStorageLimit); + } //-- try till parent present if (parentDomainJoinVO.getParent() != null && parentDomainJoinVO.getParent() != Domain.ROOT_DOMAIN) { setParentResourceLimitIfNeeded(resourceLimitMap, parentDomainJoinVO, domainsCopy); diff --git a/server/src/main/java/com/cloud/api/query/dao/AccountJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/AccountJoinDaoImpl.java index 07b5c27438b..c81481dec74 100644 --- a/server/src/main/java/com/cloud/api/query/dao/AccountJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/AccountJoinDaoImpl.java @@ -220,7 +220,7 @@ public class AccountJoinDaoImpl extends GenericDaoBase impl response.setMemoryTotal(memoryTotal); response.setMemoryAvailable(memoryAvail); - //get resource limits for primary storage space and convert it from Bytes to GiB + //get resource limits for primary storage space and convert it from Bytes to GiB long primaryStorageLimit = ApiDBUtils.findCorrectResourceLimit(account.getPrimaryStorageLimit(), account.getId(), ResourceType.primary_storage); String primaryStorageLimitDisplay = (fullView || primaryStorageLimit == -1) ? Resource.UNLIMITED : String.valueOf(primaryStorageLimit / ResourceType.bytesToGiB); long primaryStorageTotal = (account.getPrimaryStorageTotal() == null) ? 0 : (account.getPrimaryStorageTotal() / ResourceType.bytesToGiB); @@ -240,6 +240,42 @@ public class AccountJoinDaoImpl extends GenericDaoBase impl response.setSecondaryStorageLimit(secondaryStorageLimitDisplay); response.setSecondaryStorageTotal(secondaryStorageTotal); response.setSecondaryStorageAvailable(secondaryStorageAvail); + + //get resource limits for backups + long backupLimit = ApiDBUtils.findCorrectResourceLimit(account.getBackupLimit(), account.getId(), ResourceType.backup); + String backupLimitDisplay = (fullView || backupLimit == -1) ? Resource.UNLIMITED : String.valueOf(backupLimit); + long backupTotal = (account.getBackupTotal() == null) ? 0 : account.getBackupTotal(); + String backupAvail = (fullView || backupLimit == -1) ? Resource.UNLIMITED : String.valueOf(backupLimit - backupTotal); + response.setBackupLimit(backupLimitDisplay); + response.setBackupTotal(backupTotal); + response.setBackupAvailable(backupAvail); + + //get resource limits for backup storage space and convert it from Bytes to GiB + long backupStorageLimit = ApiDBUtils.findCorrectResourceLimit(account.getBackupStorageLimit(), account.getId(), ResourceType.backup_storage); + String backupStorageLimitDisplay = (fullView || backupStorageLimit == -1) ? Resource.UNLIMITED : String.valueOf(backupStorageLimit / ResourceType.bytesToGiB); + long backupStorageTotal = (account.getBackupStorageTotal() == null) ? 0 : (account.getBackupStorageTotal() / ResourceType.bytesToGiB); + String backupStorageAvail = (fullView || backupStorageLimit == -1) ? Resource.UNLIMITED : String.valueOf((backupStorageLimit / ResourceType.bytesToGiB) - backupStorageTotal); + response.setBackupStorageLimit(backupStorageLimitDisplay); + response.setBackupStorageTotal(backupStorageTotal); + response.setBackupStorageAvailable(backupStorageAvail); + + //get resource limits for buckets + long bucketLimit = ApiDBUtils.findCorrectResourceLimit(account.getBucketLimit(), account.getId(), ResourceType.bucket); + String bucketLimitDisplay = (fullView || bucketLimit == -1) ? Resource.UNLIMITED : String.valueOf(bucketLimit); + long bucketTotal = (account.getBucketTotal() == null) ? 0 : account.getBucketTotal(); + String bucketAvail = (fullView || bucketLimit == -1) ? Resource.UNLIMITED : String.valueOf(bucketLimit - bucketTotal); + response.setBucketLimit(bucketLimitDisplay); + response.setBucketTotal(bucketTotal); + response.setBucketAvailable(bucketAvail); + + //get resource limits for object storage space and convert it from Bytes to GiB + long objectStorageLimit = ApiDBUtils.findCorrectResourceLimit(account.getObjectStorageLimit(), account.getId(), ResourceType.object_storage); + String objectStorageLimitDisplay = (fullView || objectStorageLimit == -1) ? Resource.UNLIMITED : String.valueOf(objectStorageLimit / ResourceType.bytesToGiB); + long objectStorageTotal = (account.getObjectStorageTotal() == null) ? 0 : (account.getObjectStorageTotal() / ResourceType.bytesToGiB); + String objectStorageAvail = (fullView || objectStorageLimit == -1) ? Resource.UNLIMITED : String.valueOf((objectStorageLimit / ResourceType.bytesToGiB) - objectStorageTotal); + response.setObjectStorageLimit(objectStorageLimitDisplay); + response.setObjectStorageTotal(objectStorageTotal); + response.setObjectStorageAvailable(objectStorageAvail); } @Override diff --git a/server/src/main/java/com/cloud/api/query/dao/DomainJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/DomainJoinDaoImpl.java index 9ad05d216a9..79376a37b97 100644 --- a/server/src/main/java/com/cloud/api/query/dao/DomainJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/DomainJoinDaoImpl.java @@ -212,6 +212,42 @@ public class DomainJoinDaoImpl extends GenericDaoBase implem response.setSecondaryStorageLimit(secondaryStorageLimitDisplay); response.setSecondaryStorageTotal(secondaryStorageTotal); response.setSecondaryStorageAvailable(secondaryStorageAvail); + + //get resource limits for backups + long backupLimit = ApiDBUtils.findCorrectResourceLimitForDomain(domain.getBackupLimit(), ResourceType.backup, domain.getId()); + String backupLimitDisplay = (fullView || snapshotLimit == -1) ? Resource.UNLIMITED : String.valueOf(backupLimit); + long backupTotal = (domain.getBackupTotal() == null) ? 0 : domain.getBackupTotal(); + String backupAvail = (fullView || snapshotLimit == -1) ? Resource.UNLIMITED : String.valueOf(backupLimit - backupTotal); + response.setBackupLimit(backupLimitDisplay); + response.setBackupTotal(backupTotal); + response.setBackupAvailable(backupAvail); + + //get resource limits for backup storage space and convert it from Bytes to GiB + long backupStorageLimit = ApiDBUtils.findCorrectResourceLimitForDomain(domain.getBackupStorageLimit(), ResourceType.backup_storage, domain.getId()); + String backupStorageLimitDisplay = (fullView || backupLimit == -1) ? Resource.UNLIMITED : String.valueOf(backupStorageLimit / ResourceType.bytesToGiB); + long backupStorageTotal = (domain.getBackupStorageTotal() == null) ? 0 : (domain.getBackupStorageTotal() / ResourceType.bytesToGiB); + String backupStorageAvail = (fullView || backupStorageLimit == -1) ? Resource.UNLIMITED : String.valueOf((backupStorageLimit / ResourceType.bytesToGiB) - backupStorageTotal); + response.setBackupStorageLimit(backupStorageLimitDisplay); + response.setBackupStorageTotal(backupStorageTotal); + response.setBackupStorageAvailable(backupStorageAvail); + + //get resource limits for buckets + long bucketLimit = ApiDBUtils.findCorrectResourceLimit(domain.getBucketLimit(), domain.getId(), ResourceType.bucket); + String bucketLimitDisplay = (fullView || bucketLimit == -1) ? Resource.UNLIMITED : String.valueOf(bucketLimit); + long bucketTotal = (domain.getBucketTotal() == null) ? 0 : domain.getBucketTotal(); + String bucketAvail = (fullView || bucketLimit == -1) ? Resource.UNLIMITED : String.valueOf(bucketLimit - bucketTotal); + response.setBucketLimit(bucketLimitDisplay); + response.setBucketTotal(bucketTotal); + response.setBucketAvailable(bucketAvail); + + //get resource limits for object storage space and convert it from Bytes to GiB + long objectStorageLimit = ApiDBUtils.findCorrectResourceLimit(domain.getObjectStorageLimit(), domain.getId(), ResourceType.object_storage); + String objectStorageLimitDisplay = (fullView || objectStorageLimit == -1) ? Resource.UNLIMITED : String.valueOf(objectStorageLimit / ResourceType.bytesToGiB); + long objectStorageTotal = (domain.getObjectStorageTotal() == null) ? 0 : (domain.getObjectStorageTotal() / ResourceType.bytesToGiB); + String objectStorageAvail = (fullView || objectStorageLimit == -1) ? Resource.UNLIMITED : String.valueOf((objectStorageLimit / ResourceType.bytesToGiB) - objectStorageTotal); + response.setObjectStorageLimit(objectStorageLimitDisplay); + response.setObjectStorageTotal(objectStorageTotal); + response.setObjectStorageAvailable(objectStorageAvail); } @Override diff --git a/server/src/main/java/com/cloud/api/query/dao/SnapshotJoinDao.java b/server/src/main/java/com/cloud/api/query/dao/SnapshotJoinDao.java index 4e916e66ae7..25dfbfe6714 100644 --- a/server/src/main/java/com/cloud/api/query/dao/SnapshotJoinDao.java +++ b/server/src/main/java/com/cloud/api/query/dao/SnapshotJoinDao.java @@ -23,10 +23,7 @@ import org.apache.cloudstack.api.ResponseObject; import org.apache.cloudstack.api.response.SnapshotResponse; import com.cloud.api.query.vo.SnapshotJoinVO; -import com.cloud.utils.Pair; -import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDao; -import com.cloud.utils.db.SearchCriteria; public interface SnapshotJoinDao extends GenericDao { @@ -34,8 +31,7 @@ public interface SnapshotJoinDao extends GenericDao { SnapshotResponse setSnapshotResponse(SnapshotResponse snapshotResponse, SnapshotJoinVO snapshot); - Pair, Integer> searchIncludingRemovedAndCount(final SearchCriteria sc, final Filter filter); - List searchBySnapshotStorePair(String... pairs); + List findByDistinctIds(Long zoneId, Long... ids); } diff --git a/server/src/main/java/com/cloud/api/query/dao/SnapshotJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/SnapshotJoinDaoImpl.java index 04816ec87b9..2944e69c22a 100644 --- a/server/src/main/java/com/cloud/api/query/dao/SnapshotJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/SnapshotJoinDaoImpl.java @@ -18,6 +18,8 @@ package com.cloud.api.query.dao; import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -34,6 +36,8 @@ import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.query.QueryService; +import org.apache.commons.collections.CollectionUtils; + import com.cloud.api.ApiDBUtils; import com.cloud.api.ApiResponseHelper; import com.cloud.api.query.vo.SnapshotJoinVO; @@ -44,7 +48,6 @@ import com.cloud.storage.Volume.Type; import com.cloud.storage.VolumeVO; import com.cloud.user.Account; import com.cloud.user.AccountService; -import com.cloud.utils.Pair; import com.cloud.utils.db.Filter; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; @@ -53,11 +56,11 @@ import com.cloud.vm.VMInstanceVO; public class SnapshotJoinDaoImpl extends GenericDaoBaseWithTagInformation implements SnapshotJoinDao { @Inject - private AccountService accountService; + AccountService accountService; @Inject - private AnnotationDao annotationDao; + AnnotationDao annotationDao; @Inject - private ConfigurationDao configDao; + ConfigurationDao configDao; @Inject SnapshotDataFactory snapshotDataFactory; @@ -85,7 +88,7 @@ public class SnapshotJoinDaoImpl extends GenericDaoBaseWithTagInformation, Integer> searchIncludingRemovedAndCount(final SearchCriteria sc, final Filter filter) { - List objects = searchIncludingRemoved(sc, filter, null, false); - Integer count = getDistinctCount(sc); - return new Pair<>(objects, count); - } - @Override public List searchBySnapshotStorePair(String... pairs) { // set detail batch query size @@ -243,14 +239,33 @@ public class SnapshotJoinDaoImpl extends GenericDaoBaseWithTagInformation findById(Long zoneId, long id) { + SearchBuilder sb = createSearchBuilder(); + sb.and("id", sb.entity().getId(), SearchCriteria.Op.EQ); + sb.and("zoneId", sb.entity().getDataCenterId(), SearchCriteria.Op.EQ); + sb.done(); + SearchCriteria sc = sb.create(); + sc.setParameters("id", id); + if (zoneId != null) { + sc.setParameters("zoneId", zoneId); + } + List snapshotJoinVOS = search(sc, null); + if (CollectionUtils.isEmpty(snapshotJoinVOS)) { + return null; + } + snapshotJoinVOS.sort(Comparator.comparing(SnapshotJoinVO::getSnapshotStorePair)); + return Collections.singletonList(snapshotJoinVOS.get(0)); + } + @Override public List findByDistinctIds(Long zoneId, Long... ids) { if (ids == null || ids.length == 0) { return new ArrayList<>(); } - + if (ids.length == 1) { + return findById(zoneId, ids[0]); + } Filter searchFilter = new Filter(SnapshotJoinVO.class, "snapshotStorePair", QueryService.SortKeyAscending.value(), null, null); - SearchCriteria sc = snapshotIdsSearch.create(); if (zoneId != null) { sc.setParameters("zoneId", zoneId); diff --git a/server/src/main/java/com/cloud/api/query/vo/AccountJoinVO.java b/server/src/main/java/com/cloud/api/query/vo/AccountJoinVO.java index 0bd28d2af32..2e39816ed41 100644 --- a/server/src/main/java/com/cloud/api/query/vo/AccountJoinVO.java +++ b/server/src/main/java/com/cloud/api/query/vo/AccountJoinVO.java @@ -123,6 +123,18 @@ public class AccountJoinVO extends BaseViewVO implements InternalIdentity, Ident @Column(name = "snapshotTotal") private Long snapshotTotal; + @Column(name = "backupLimit") + private Long backupLimit; + + @Column(name = "backupTotal") + private Long backupTotal; + + @Column(name = "backupStorageLimit") + private Long backupStorageLimit; + + @Column(name = "backupStorageTotal") + private Long backupStorageTotal; + @Column(name = "templateLimit") private Long templateLimit; @@ -177,6 +189,18 @@ public class AccountJoinVO extends BaseViewVO implements InternalIdentity, Ident @Column(name = "secondaryStorageTotal") private Long secondaryStorageTotal; + @Column(name = "bucketLimit") + private Long bucketLimit; + + @Column(name = "bucketTotal") + private Long bucketTotal; + + @Column(name = "objectStorageLimit") + private Long objectStorageLimit; + + @Column(name = "objectStorageTotal") + private Long objectStorageTotal; + @Column(name = "job_id") private Long jobId; @@ -293,6 +317,14 @@ public class AccountJoinVO extends BaseViewVO implements InternalIdentity, Ident return snapshotTotal; } + public Long getBackupTotal() { + return backupTotal; + } + + public Long getBackupStorageTotal() { + return backupStorageTotal; + } + public Long getTemplateTotal() { return templateTotal; } @@ -333,6 +365,14 @@ public class AccountJoinVO extends BaseViewVO implements InternalIdentity, Ident return secondaryStorageTotal; } + public Long getBucketTotal() { + return bucketTotal; + } + + public Long getObjectStorageTotal() { + return objectStorageTotal; + } + public Long getVmLimit() { return vmLimit; } @@ -349,6 +389,14 @@ public class AccountJoinVO extends BaseViewVO implements InternalIdentity, Ident return snapshotLimit; } + public Long getBackupLimit() { + return backupLimit; + } + + public Long getBackupStorageLimit() { + return backupStorageLimit; + } + public Long getTemplateLimit() { return templateLimit; } @@ -381,6 +429,14 @@ public class AccountJoinVO extends BaseViewVO implements InternalIdentity, Ident return secondaryStorageLimit; } + public Long getBucketLimit() { + return bucketLimit; + } + + public Long getObjectStorageLimit() { + return objectStorageLimit; + } + public Long getJobId() { return jobId; } diff --git a/server/src/main/java/com/cloud/api/query/vo/DomainJoinVO.java b/server/src/main/java/com/cloud/api/query/vo/DomainJoinVO.java index e17eacd68fa..3e623690f10 100644 --- a/server/src/main/java/com/cloud/api/query/vo/DomainJoinVO.java +++ b/server/src/main/java/com/cloud/api/query/vo/DomainJoinVO.java @@ -100,6 +100,18 @@ public class DomainJoinVO extends BaseViewVO implements InternalIdentity, Identi @Column(name="snapshotTotal") private Long snapshotTotal; + @Column(name="backupLimit") + private Long backupLimit; + + @Column(name = "backupStorageLimit") + private Long backupStorageLimit; + + @Column(name = "backupStorageTotal") + private Long backupStorageTotal; + + @Column(name="backupTotal") + private Long backupTotal; + @Column(name="templateLimit") private Long templateLimit; @@ -154,6 +166,18 @@ public class DomainJoinVO extends BaseViewVO implements InternalIdentity, Identi @Column(name="secondaryStorageTotal") private Long secondaryStorageTotal; + @Column(name = "bucketLimit") + private Long bucketLimit; + + @Column(name = "bucketTotal") + private Long bucketTotal; + + @Column(name = "objectStorageLimit") + private Long objectStorageLimit; + + @Column(name = "objectStorageTotal") + private Long objectStorageTotal; + @Transient private String parentName; @@ -311,8 +335,13 @@ public class DomainJoinVO extends BaseViewVO implements InternalIdentity, Identi this.snapshotTotal = snapshotTotal; } + public Long getBackupTotal() { + return backupTotal; + } - + public Long getBackupStorageTotal() { + return backupStorageTotal; + } public Long getTemplateTotal() { return templateTotal; @@ -393,6 +422,13 @@ public class DomainJoinVO extends BaseViewVO implements InternalIdentity, Identi this.secondaryStorageTotal = secondaryStorageTotal; } + public Long getBucketTotal() { + return bucketTotal; + } + + public Long getObjectStorageTotal() { + return objectStorageTotal; + } public Long getVmLimit() { return vmLimit; @@ -433,6 +469,21 @@ public class DomainJoinVO extends BaseViewVO implements InternalIdentity, Identi this.snapshotLimit = snapshotLimit; } + public Long getBackupLimit() { + return backupLimit; + } + + public void setBackupLimit(Long backupLimit) { + this.backupLimit = backupLimit; + } + + public Long getBackupStorageLimit() { + return backupStorageLimit; + } + + public void setBackupStorageLimit(Long backupStorageLimit) { + this.backupStorageLimit = backupStorageLimit; + } public Long getTemplateLimit() { return templateLimit; @@ -513,6 +564,22 @@ public class DomainJoinVO extends BaseViewVO implements InternalIdentity, Identi this.secondaryStorageLimit = secondaryStorageLimit; } + public Long getBucketLimit() { + return bucketLimit; + } + + public void setBucketLimit(Long bucketLimit) { + this.bucketLimit = bucketLimit; + } + + public Long getObjectStorageLimit() { + return objectStorageLimit; + } + + public void setObjectStorageLimit(Long objectStorageLimit) { + this.objectStorageLimit = objectStorageLimit; + } + public String getParentName() { return parentName; } diff --git a/server/src/main/java/com/cloud/configuration/Config.java b/server/src/main/java/com/cloud/configuration/Config.java index b9de906ba46..de882b4cf01 100644 --- a/server/src/main/java/com/cloud/configuration/Config.java +++ b/server/src/main/java/com/cloud/configuration/Config.java @@ -19,7 +19,6 @@ package com.cloud.configuration; import java.util.ArrayList; import java.util.HashMap; import java.util.List; -import java.util.StringTokenizer; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; import org.apache.cloudstack.engine.subsystem.api.storage.StoragePoolAllocator; @@ -506,7 +505,7 @@ public enum Config { "The time interval in seconds when the management server polls for snapshots to be scheduled.", null), SnapshotDeltaMax("Snapshots", SnapshotManager.class, Integer.class, "snapshot.delta.max", "16", "max delta snapshots between two full snapshots.", null), - KVMSnapshotEnabled("Hidden", SnapshotManager.class, Boolean.class, "kvm.snapshot.enabled", "false", "whether snapshot is enabled for KVM hosts", null), + KVMSnapshotEnabled("Hidden", SnapshotManager.class, Boolean.class, "kvm.snapshot.enabled", "false", "Whether volume snapshot is enabled on running instances on a KVM host", null), // Advanced EventPurgeInterval( @@ -665,8 +664,8 @@ public enum Config { ManagementServer.class, String.class, "hypervisor.list", - HypervisorType.Hyperv + "," + HypervisorType.KVM + "," + HypervisorType.XenServer + "," + HypervisorType.VMware + "," + HypervisorType.BareMetal + "," + - HypervisorType.Ovm + "," + HypervisorType.LXC + "," + HypervisorType.Ovm3, + HypervisorType.KVM + "," + HypervisorType.VMware + "," + HypervisorType.XenServer + "," + HypervisorType.Hyperv + "," + + HypervisorType.BareMetal + "," + HypervisorType.Ovm + "," + HypervisorType.LXC + "," + HypervisorType.Ovm3, "The list of hypervisors that this deployment will use.", "hypervisorList", ConfigKey.Kind.CSV, @@ -1365,7 +1364,7 @@ public enum Config { "200", "The default maximum primary storage space (in GiB) that can be used for an account", null), -DefaultMaxAccountProjects( + DefaultMaxAccountProjects( "Account Defaults", ManagementServer.class, Long.class, @@ -1807,26 +1806,25 @@ DefaultMaxAccountProjects( private final String _defaultValue; private final String _description; private final String _range; - private final String _scope; // Parameter can be at different levels (Zone/cluster/pool/account), by default every parameter is at global + private final int _scope; // Parameter can be at different levels (Zone/cluster/pool/account), by default every parameter is at global private final ConfigKey.Kind _kind; private final String _options; - private static final HashMap> s_scopeLevelConfigsMap = new HashMap>(); + private static final HashMap> s_scopeLevelConfigsMap = new HashMap<>(); static { - s_scopeLevelConfigsMap.put(ConfigKey.Scope.Zone.toString(), new ArrayList()); - s_scopeLevelConfigsMap.put(ConfigKey.Scope.Cluster.toString(), new ArrayList()); - s_scopeLevelConfigsMap.put(ConfigKey.Scope.StoragePool.toString(), new ArrayList()); - s_scopeLevelConfigsMap.put(ConfigKey.Scope.Account.toString(), new ArrayList()); - s_scopeLevelConfigsMap.put(ConfigKey.Scope.Global.toString(), new ArrayList()); + s_scopeLevelConfigsMap.put(ConfigKey.Scope.Zone.getBitValue(), new ArrayList()); + s_scopeLevelConfigsMap.put(ConfigKey.Scope.Cluster.getBitValue(), new ArrayList()); + s_scopeLevelConfigsMap.put(ConfigKey.Scope.StoragePool.getBitValue(), new ArrayList()); + s_scopeLevelConfigsMap.put(ConfigKey.Scope.Account.getBitValue(), new ArrayList()); + s_scopeLevelConfigsMap.put(ConfigKey.Scope.Global.getBitValue(), new ArrayList()); for (Config c : Config.values()) { //Creating group of parameters per each level (zone/cluster/pool/account) - StringTokenizer tokens = new StringTokenizer(c.getScope(), ","); - while (tokens.hasMoreTokens()) { - String scope = tokens.nextToken().trim(); - List currentConfigs = s_scopeLevelConfigsMap.get(scope); + List scopes = ConfigKey.Scope.decode(c.getScope()); + for (ConfigKey.Scope scope : scopes) { + List currentConfigs = s_scopeLevelConfigsMap.get(scope.getBitValue()); currentConfigs.add(c); - s_scopeLevelConfigsMap.put(scope, currentConfigs); + s_scopeLevelConfigsMap.put(scope.getBitValue(), currentConfigs); } } } @@ -1870,7 +1868,7 @@ DefaultMaxAccountProjects( _defaultValue = defaultValue; _description = description; _range = range; - _scope = ConfigKey.Scope.Global.toString(); + _scope = ConfigKey.Scope.Global.getBitValue(); _kind = kind; _options = options; } @@ -1895,7 +1893,7 @@ DefaultMaxAccountProjects( return _type; } - public String getScope() { + public int getScope() { return _scope; } diff --git a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java index 1a803f1a88b..0d81991aa13 100644 --- a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java +++ b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java @@ -1087,7 +1087,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati Optional optionalValue; String defaultValue; String category; - String configScope; + List configScope; final ConfigurationVO config = _configDao.findByName(name); if (config == null) { configKey = _configDepot.get(name); @@ -1097,11 +1097,11 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati } defaultValue = configKey.defaultValue(); category = configKey.category(); - configScope = configKey.scope().toString(); + configScope = configKey.getScopes(); } else { defaultValue = config.getDefaultValue(); category = config.getCategory(); - configScope = config.getScope(); + configScope = config.getScopes(); } String scope = ""; @@ -1126,8 +1126,11 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati throw new InvalidParameterValueException("cannot handle multiple IDs, provide only one ID corresponding to the scope"); } - if (scope != null && !scope.equals(ConfigKey.Scope.Global.toString()) && !configScope.contains(scope)) { - throw new InvalidParameterValueException("Invalid scope id provided for the parameter " + name); + if (scope != null) { + ConfigKey.Scope scopeVal = ConfigKey.Scope.valueOf(scope); + if (!scope.equals(ConfigKey.Scope.Global.toString()) && !configScope.contains(scopeVal)) { + throw new InvalidParameterValueException("Invalid scope id provided for the parameter " + name); + } } String newValue = null; @@ -1237,10 +1240,11 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati return "Invalid configuration variable."; } - String configScope = cfg.getScope(); + List configScope = cfg.getScopes(); if (scope != null) { - if (!configScope.contains(scope) && - !(ENABLE_ACCOUNT_SETTINGS_FOR_DOMAIN.value() && configScope.contains(ConfigKey.Scope.Account.toString()) && + ConfigKey.Scope scopeVal = ConfigKey.Scope.valueOf(scope); + if (!configScope.contains(scopeVal) && + !(ENABLE_ACCOUNT_SETTINGS_FOR_DOMAIN.value() && configScope.contains(ConfigKey.Scope.Account) && scope.equals(ConfigKey.Scope.Domain.toString()))) { logger.error("Invalid scope id provided for the parameter " + name); return "Invalid scope id provided for the parameter " + name; @@ -7240,10 +7244,6 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati sc.addAnd("id", SearchCriteria.Op.EQ, id); } - if (tags != null) { - sc.addAnd("tags", SearchCriteria.Op.EQ, tags); - } - if (isTagged != null) { if (isTagged) { sc.addAnd("tags", SearchCriteria.Op.NNULL); @@ -7252,6 +7252,17 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati } } + if (tags != null) { + if (GuestType.Shared.name().equalsIgnoreCase(guestIpType)) { + SearchCriteria tagsSc = networkOfferingJoinDao.createSearchCriteria(); + tagsSc.addAnd("tags", SearchCriteria.Op.EQ, tags); + tagsSc.addOr("isDefault", SearchCriteria.Op.EQ, true); + sc.addAnd("tags", SearchCriteria.Op.SC, tagsSc); + } else { + sc.addAnd("tags", SearchCriteria.Op.EQ, tags); + } + } + if (zoneId != null) { SearchBuilder sb = networkOfferingJoinDao.createSearchBuilder(); sb.and("zoneId", sb.entity().getZoneId(), SearchCriteria.Op.FIND_IN_SET); @@ -7312,7 +7323,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati boolean addOffering = true; List checkForProviders = new ArrayList(); - if (checkForTags && ! checkNetworkOfferingTags(pNtwkTags, allowNullTag, offering.getTags())) { + if (checkForTags && !checkNetworkOfferingTags(pNtwkTags, allowNullTag, offering.getTags())) { continue; } diff --git a/server/src/main/java/com/cloud/network/IpAddressManagerImpl.java b/server/src/main/java/com/cloud/network/IpAddressManagerImpl.java index 1fa416b643b..ea8a66ecf9f 100644 --- a/server/src/main/java/com/cloud/network/IpAddressManagerImpl.java +++ b/server/src/main/java/com/cloud/network/IpAddressManagerImpl.java @@ -2476,7 +2476,7 @@ public class IpAddressManagerImpl extends ManagerBase implements IpAddressManage @Override public PublicIpQuarantine addPublicIpAddressToQuarantine(IpAddress publicIpAddress, Long domainId) { - Integer quarantineDuration = PUBLIC_IP_ADDRESS_QUARANTINE_DURATION.valueInDomain(domainId); + Integer quarantineDuration = PUBLIC_IP_ADDRESS_QUARANTINE_DURATION.valueIn(domainId); if (quarantineDuration <= 0) { logger.debug(String.format("Not adding IP [%s] to quarantine because configuration [%s] has value equal or less to 0.", publicIpAddress.getAddress(), PUBLIC_IP_ADDRESS_QUARANTINE_DURATION.key())); diff --git a/server/src/main/java/com/cloud/network/NetworkModelImpl.java b/server/src/main/java/com/cloud/network/NetworkModelImpl.java index 5f6cae31699..6cc18aaef54 100644 --- a/server/src/main/java/com/cloud/network/NetworkModelImpl.java +++ b/server/src/main/java/com/cloud/network/NetworkModelImpl.java @@ -934,8 +934,12 @@ public class NetworkModelImpl extends ManagerBase implements NetworkModel, Confi @Override public UserDataServiceProvider getUserDataUpdateProvider(Network network) { - String userDataProvider = _ntwkSrvcDao.getProviderForServiceInNetwork(network.getId(), Service.UserData); + if (network == null) { + logger.warn("No network details, can't fetch user data provider"); + return null; + } + String userDataProvider = _ntwkSrvcDao.getProviderForServiceInNetwork(network.getId(), Service.UserData); if (userDataProvider == null) { logger.debug("Network " + network + " doesn't support service " + Service.UserData.getName()); return null; diff --git a/server/src/main/java/com/cloud/network/element/ConfigDriveNetworkElement.java b/server/src/main/java/com/cloud/network/element/ConfigDriveNetworkElement.java index 6ef84a04b9f..e1f850978d5 100644 --- a/server/src/main/java/com/cloud/network/element/ConfigDriveNetworkElement.java +++ b/server/src/main/java/com/cloud/network/element/ConfigDriveNetworkElement.java @@ -325,8 +325,12 @@ public class ConfigDriveNetworkElement extends AdapterBase implements NetworkEle try { final Network network = _networkMgr.getNetwork(nic.getNetworkId()); final UserDataServiceProvider userDataUpdateProvider = _networkModel.getUserDataUpdateProvider(network); + if (userDataUpdateProvider == null) { + logger.warn("Failed to get user data provider"); + return false; + } final Provider provider = userDataUpdateProvider.getProvider(); - if (provider.equals(Provider.ConfigDrive)) { + if (Provider.ConfigDrive.equals(provider)) { try { return deleteConfigDriveIso(vm); } catch (ResourceUnavailableException e) { @@ -341,8 +345,13 @@ public class ConfigDriveNetworkElement extends AdapterBase implements NetworkEle @Override public boolean prepareMigration(NicProfile nic, Network network, VirtualMachineProfile vm, DeployDestination dest, ReservationContext context) { - if (_networkModel.getUserDataUpdateProvider(network).getProvider().equals(Provider.ConfigDrive)) { - logger.trace(String.format("[prepareMigration] for vm: %s", vm)); + final UserDataServiceProvider userDataUpdateProvider = _networkModel.getUserDataUpdateProvider(network); + if (userDataUpdateProvider == null) { + logger.warn("Failed to prepare for migration, can't get user data provider"); + return false; + } + if (Provider.ConfigDrive.equals(userDataUpdateProvider.getProvider())) { + logger.trace(String.format("[prepareMigration] for vm: %s", vm.getInstanceName())); try { if (isConfigDriveIsoOnHostCache(vm.getId())) { vm.setConfigDriveLocation(Location.HOST); @@ -392,7 +401,11 @@ public class ConfigDriveNetworkElement extends AdapterBase implements NetworkEle } private void recreateConfigDriveIso(NicProfile nic, Network network, VirtualMachineProfile vm, DeployDestination dest) throws ResourceUnavailableException { - if (nic.isDefaultNic() && _networkModel.getUserDataUpdateProvider(network).getProvider().equals(Provider.ConfigDrive)) { + final UserDataServiceProvider userDataUpdateProvider = _networkModel.getUserDataUpdateProvider(network); + if (userDataUpdateProvider == null) { + return; + } + if (nic.isDefaultNic() && Provider.ConfigDrive.equals(userDataUpdateProvider.getProvider())) { DiskTO diskToUse = null; for (DiskTO disk : vm.getDisks()) { if (disk.getType() == Volume.Type.ISO && disk.getPath() != null && disk.getPath().contains("configdrive")) { diff --git a/server/src/main/java/com/cloud/network/element/VirtualRouterElement.java b/server/src/main/java/com/cloud/network/element/VirtualRouterElement.java index d4aeae23a2d..263ff523ab6 100644 --- a/server/src/main/java/com/cloud/network/element/VirtualRouterElement.java +++ b/server/src/main/java/com/cloud/network/element/VirtualRouterElement.java @@ -769,7 +769,12 @@ NetworkMigrationResponder, AggregatedCommandExecutor, RedundantResource, DnsServ @Override public boolean saveHypervisorHostname(NicProfile nicProfile, Network network, VirtualMachineProfile vm, DeployDestination dest) throws ResourceUnavailableException { - if (_networkModel.getUserDataUpdateProvider(network).getProvider().equals(Provider.VirtualRouter) && vm.getVirtualMachine().getType() == VirtualMachine.Type.User) { + final UserDataServiceProvider userDataUpdateProvider = _networkModel.getUserDataUpdateProvider(network); + if (userDataUpdateProvider == null) { + logger.warn("Failed to update hypervisor host details, can't get user data provider"); + return false; + } + if (Provider.VirtualRouter.equals(userDataUpdateProvider.getProvider()) && vm.getVirtualMachine().getType() == VirtualMachine.Type.User) { VirtualMachine uvm = vm.getVirtualMachine(); UserVmVO destVm = _userVmDao.findById(uvm.getId()); VirtualMachineProfile profile = null; diff --git a/server/src/main/java/com/cloud/network/firewall/FirewallManagerImpl.java b/server/src/main/java/com/cloud/network/firewall/FirewallManagerImpl.java index 6b8133534d2..4aee5fef48a 100644 --- a/server/src/main/java/com/cloud/network/firewall/FirewallManagerImpl.java +++ b/server/src/main/java/com/cloud/network/firewall/FirewallManagerImpl.java @@ -1009,7 +1009,7 @@ public class FirewallManagerImpl extends ManagerBase implements FirewallService, Long relatedRuleId, long networkId) throws NetworkRuleConflictException { // If firwallRule for this port range already exists, return it - List rules = _firewallDao.listByIpPurposeAndProtocolAndNotRevoked(ipAddrId, startPort, endPort, protocol, Purpose.Firewall); + List rules = _firewallDao.listByIpPurposePortsProtocolAndNotRevoked(ipAddrId, startPort, endPort, protocol, Purpose.Firewall); if (!rules.isEmpty()) { return rules.get(0); } diff --git a/server/src/main/java/com/cloud/network/vpn/Site2SiteVpnManagerImpl.java b/server/src/main/java/com/cloud/network/vpn/Site2SiteVpnManagerImpl.java index ed83e396abf..8be97644bf5 100644 --- a/server/src/main/java/com/cloud/network/vpn/Site2SiteVpnManagerImpl.java +++ b/server/src/main/java/com/cloud/network/vpn/Site2SiteVpnManagerImpl.java @@ -343,7 +343,7 @@ public class Site2SiteVpnManagerImpl extends ManagerBase implements Site2SiteVpn private void validatePrerequisiteVpnGateway(Site2SiteVpnGateway vpnGateway) { // check if gateway has been defined on the VPC if (_vpnGatewayDao.findByVpcId(vpnGateway.getVpcId()) == null) { - throw new InvalidParameterValueException("we can not create a VPN connection for a VPC that does not have a VPN gateway defined"); + throw new InvalidParameterValueException("We can not create a VPN connection for a VPC that does not have a VPN gateway defined"); } } @@ -590,7 +590,7 @@ public class Site2SiteVpnManagerImpl extends ManagerBase implements Site2SiteVpn private void stopVpnConnection(Long id) throws ResourceUnavailableException { Site2SiteVpnConnectionVO conn = _vpnConnectionDao.acquireInLockTable(id); if (conn == null) { - throw new CloudRuntimeException("Unable to acquire lock for stopping of VPN connection with ID " + id); + throw new CloudRuntimeException("Unable to acquire lock for stopping VPN connection with ID " + id); } try { if (conn.getState() == State.Pending) { diff --git a/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java b/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java index b8c6e29c278..6023ae2154b 100644 --- a/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java +++ b/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java @@ -36,12 +36,17 @@ import java.util.stream.Stream; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.event.ActionEventUtils; +import com.cloud.event.EventTypes; import com.cloud.utils.Ternary; import org.apache.cloudstack.acl.SecurityChecker.AccessType; +import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.response.AccountResponse; import org.apache.cloudstack.api.response.DomainResponse; import org.apache.cloudstack.api.response.ResourceLimitAndCountResponse; import org.apache.cloudstack.api.response.TaggedResourceLimitAndCountResponse; +import org.apache.cloudstack.backup.BackupManager; +import org.apache.cloudstack.backup.dao.BackupDao; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; import org.apache.cloudstack.framework.config.ConfigKey; @@ -55,6 +60,7 @@ import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; +import org.apache.cloudstack.storage.object.BucketApiService; import org.apache.cloudstack.utils.identity.ManagementServerNode; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; @@ -100,6 +106,7 @@ import com.cloud.storage.SnapshotVO; import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.BucketDao; import com.cloud.storage.dao.DiskOfferingDao; import com.cloud.storage.dao.SnapshotDao; import com.cloud.storage.dao.VMTemplateDao; @@ -170,6 +177,8 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim @Inject protected SnapshotDao _snapshotDao; @Inject + protected BackupDao backupDao; + @Inject private SnapshotDataStoreDao _snapshotDataStoreDao; @Inject private TemplateDataStoreDao _vmTemplateStoreDao; @@ -193,6 +202,8 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim ServiceOfferingDao serviceOfferingDao; @Inject DiskOfferingDao diskOfferingDao; + @Inject + BucketDao bucketDao; protected GenericSearchBuilder templateSizeSearch; protected GenericSearchBuilder snapshotSizeSearch; @@ -288,6 +299,10 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim projectResourceLimitMap.put(Resource.ResourceType.memory.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxProjectMemory.key()))); projectResourceLimitMap.put(Resource.ResourceType.primary_storage.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxProjectPrimaryStorage.key()))); projectResourceLimitMap.put(Resource.ResourceType.secondary_storage.name(), MaxProjectSecondaryStorage.value()); + projectResourceLimitMap.put(Resource.ResourceType.backup.name(), Long.parseLong(_configDao.getValue(BackupManager.DefaultMaxProjectBackups.key()))); + projectResourceLimitMap.put(Resource.ResourceType.backup_storage.name(), Long.parseLong(_configDao.getValue(BackupManager.DefaultMaxProjectBackupStorage.key()))); + projectResourceLimitMap.put(Resource.ResourceType.bucket.name(), Long.parseLong(_configDao.getValue(BucketApiService.DefaultMaxProjectBuckets.key()))); + projectResourceLimitMap.put(Resource.ResourceType.object_storage.name(), Long.parseLong(_configDao.getValue(BucketApiService.DefaultMaxProjectObjectStorage.key()))); accountResourceLimitMap.put(Resource.ResourceType.public_ip.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxAccountPublicIPs.key()))); accountResourceLimitMap.put(Resource.ResourceType.snapshot.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxAccountSnapshots.key()))); @@ -301,6 +316,10 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim accountResourceLimitMap.put(Resource.ResourceType.primary_storage.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxAccountPrimaryStorage.key()))); accountResourceLimitMap.put(Resource.ResourceType.secondary_storage.name(), MaxAccountSecondaryStorage.value()); accountResourceLimitMap.put(Resource.ResourceType.project.name(), DefaultMaxAccountProjects.value()); + accountResourceLimitMap.put(Resource.ResourceType.backup.name(), Long.parseLong(_configDao.getValue(BackupManager.DefaultMaxAccountBackups.key()))); + accountResourceLimitMap.put(Resource.ResourceType.backup_storage.name(), Long.parseLong(_configDao.getValue(BackupManager.DefaultMaxAccountBackupStorage.key()))); + accountResourceLimitMap.put(Resource.ResourceType.bucket.name(), Long.parseLong(_configDao.getValue(BucketApiService.DefaultMaxAccountBuckets.key()))); + accountResourceLimitMap.put(Resource.ResourceType.object_storage.name(), Long.parseLong(_configDao.getValue(BucketApiService.DefaultMaxAccountObjectStorage.key()))); domainResourceLimitMap.put(Resource.ResourceType.public_ip.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxDomainPublicIPs.key()))); domainResourceLimitMap.put(Resource.ResourceType.snapshot.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxDomainSnapshots.key()))); @@ -314,6 +333,10 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim domainResourceLimitMap.put(Resource.ResourceType.primary_storage.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxDomainPrimaryStorage.key()))); domainResourceLimitMap.put(Resource.ResourceType.secondary_storage.name(), Long.parseLong(_configDao.getValue(Config.DefaultMaxDomainSecondaryStorage.key()))); domainResourceLimitMap.put(Resource.ResourceType.project.name(), DefaultMaxDomainProjects.value()); + domainResourceLimitMap.put(Resource.ResourceType.backup.name(), Long.parseLong(_configDao.getValue(BackupManager.DefaultMaxDomainBackups.key()))); + domainResourceLimitMap.put(Resource.ResourceType.backup_storage.name(), Long.parseLong(_configDao.getValue(BackupManager.DefaultMaxDomainBackupStorage.key()))); + domainResourceLimitMap.put(Resource.ResourceType.bucket.name(), Long.parseLong(_configDao.getValue(BucketApiService.DefaultMaxDomainBuckets.key()))); + domainResourceLimitMap.put(Resource.ResourceType.object_storage.name(), Long.parseLong(_configDao.getValue(BucketApiService.DefaultMaxDomainObjectStorage.key()))); } catch (NumberFormatException e) { logger.error("NumberFormatException during configuration", e); throw new ConfigurationException("Configuration failed due to NumberFormatException, see log for the stacktrace"); @@ -390,8 +413,8 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim if (value < 0) { // return unlimit if value is set to negative return max; } - // convert the value from GiB to bytes in case of primary or secondary storage. - if (type == ResourceType.primary_storage || type == ResourceType.secondary_storage) { + // convert the value from GiB to bytes in case of storage type resource. + if (ResourceType.isStorageType(type)) { value = value * ResourceType.bytesToGiB; } return value; @@ -431,7 +454,7 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim if (value < 0) { // return unlimit if value is set to negative return max; } - if (type == ResourceType.primary_storage || type == ResourceType.secondary_storage) { + if (ResourceType.isStorageType(type)) { value = value * ResourceType.bytesToGiB; } return value; @@ -478,7 +501,7 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim if (value < 0) { // return unlimit if value is set to negative return max; } - if (type == ResourceType.primary_storage || type == ResourceType.secondary_storage) { + if (ResourceType.isStorageType(type)) { value = value * ResourceType.bytesToGiB; } return value; @@ -512,7 +535,7 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim String convCurrentResourceReservation = String.valueOf(currentResourceReservation); String convNumResources = String.valueOf(numResources); - if (type == ResourceType.secondary_storage || type == ResourceType.primary_storage){ + if (ResourceType.isStorageType(type)) { convDomainResourceLimit = toHumanReadableSize(domainResourceLimit); convCurrentDomainResourceCount = toHumanReadableSize(currentDomainResourceCount); convCurrentResourceReservation = toHumanReadableSize(currentResourceReservation); @@ -557,7 +580,7 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim String convertedCurrentResourceReservation = String.valueOf(currentResourceReservation); String convertedNumResources = String.valueOf(numResources); - if (type == ResourceType.secondary_storage || type == ResourceType.primary_storage){ + if (ResourceType.isStorageType(type)) { convertedAccountResourceLimit = toHumanReadableSize(accountResourceLimit); convertedCurrentResourceCount = toHumanReadableSize(currentResourceCount); convertedCurrentResourceReservation = toHumanReadableSize(currentResourceReservation); @@ -594,7 +617,7 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim public long findDefaultResourceLimitForDomain(ResourceType resourceType) { Long resourceLimit = null; resourceLimit = domainResourceLimitMap.get(resourceType.getName()); - if (resourceLimit != null && (resourceType == ResourceType.primary_storage || resourceType == ResourceType.secondary_storage)) { + if (resourceLimit != null && ResourceType.isStorageType(resourceType)) { if (! Long.valueOf(Resource.RESOURCE_UNLIMITED).equals(resourceLimit)) { resourceLimit = resourceLimit * ResourceType.bytesToGiB; } @@ -908,12 +931,14 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim } //Convert max storage size from GiB to bytes - if ((resourceType == ResourceType.primary_storage || resourceType == ResourceType.secondary_storage) && max >= 0) { + if (ResourceType.isStorageType(resourceType) && max >= 0) { max *= ResourceType.bytesToGiB; } ResourceOwnerType ownerType = null; Long ownerId = null; + ApiCommandResourceType ownerResourceType = null; + Long ownerResourceId = null; if (accountId != null) { Account account = _entityMgr.findById(Account.class, accountId); @@ -942,6 +967,16 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim ownerType = ResourceOwnerType.Account; ownerId = accountId; + + if (account.getType() == Account.Type.PROJECT) { + ownerResourceType = ApiCommandResourceType.Project; + Project project = _projectDao.findByProjectAccountId(accountId); + ownerResourceId = project.getId(); + } else { + ownerResourceType = ApiCommandResourceType.Account; + ownerResourceId = ownerId; + } + if (StringUtils.isNotEmpty(tag)) { long untaggedLimit = findCorrectResourceLimitForAccount(account, resourceType, null); if (untaggedLimit > 0 && max > untaggedLimit) { @@ -980,6 +1015,8 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim } ownerType = ResourceOwnerType.Domain; ownerId = domainId; + ownerResourceType = ApiCommandResourceType.Domain; + ownerResourceId = ownerId; } if (ownerId == null) { @@ -987,6 +1024,12 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim } ResourceLimitVO limit = _resourceLimitDao.findByOwnerIdAndTypeAndTag(ownerId, ownerType, resourceType, tag); + + ActionEventUtils.onActionEvent(caller.getId(), caller.getAccountId(), + caller.getDomainId(), EventTypes.EVENT_RESOURCE_LIMIT_UPDATE, + "Resource limit updated. Resource Type: " + resourceType.toString() + ", New Value: " + max, + ownerResourceId, ownerResourceType.toString()); + if (limit != null) { // Update the existing limit _resourceLimitDao.update(limit.getId(), max); @@ -1135,7 +1178,7 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim } if (logger.isDebugEnabled()) { String convertedDelta = String.valueOf(delta); - if (type == ResourceType.secondary_storage || type == ResourceType.primary_storage){ + if (ResourceType.isStorageType(type)) { convertedDelta = toHumanReadableSize(delta); } String typeStr = StringUtils.isNotEmpty(tag) ? String.format("%s (tag: %s)", type, tag) : type.getName(); @@ -1236,6 +1279,10 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim newCount = calculateVolumeCountForAccount(accountId, tag); } else if (type == Resource.ResourceType.snapshot) { newCount = _snapshotDao.countSnapshotsForAccount(accountId); + } else if (type == Resource.ResourceType.backup) { + newCount = backupDao.countBackupsForAccount(accountId); + } else if (type == Resource.ResourceType.backup_storage) { + newCount = backupDao.calculateBackupStorageForAccount(accountId); } else if (type == Resource.ResourceType.public_ip) { newCount = calculatePublicIpForAccount(accountId); } else if (type == Resource.ResourceType.template) { @@ -1254,6 +1301,10 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim newCount = calculatePrimaryStorageForAccount(accountId, tag); } else if (type == Resource.ResourceType.secondary_storage) { newCount = calculateSecondaryStorageForAccount(accountId); + } else if (type == Resource.ResourceType.bucket) { + newCount = bucketDao.countBucketsForAccount(accountId); + } else if (type == ResourceType.object_storage) { + newCount = bucketDao.calculateObjectStorageAllocationForAccount(accountId); } else { throw new InvalidParameterValueException("Unsupported resource type " + type); } @@ -1270,10 +1321,9 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim _resourceCountDao.persist(new ResourceCountVO(type, newCount, accountId, ResourceOwnerType.Account, tag)); } - // No need to log message for primary and secondary storage because both are recalculating the + // No need to log message for storage type resources because both are recalculating the // resource count which will not lead to any discrepancy. - if (newCount != null && !newCount.equals(oldCount) && - type != Resource.ResourceType.primary_storage && type != Resource.ResourceType.secondary_storage) { + if (newCount != null && !newCount.equals(oldCount) && !ResourceType.isStorageType(type)) { logger.warn("Discrepancy in the resource count " + "(original count=" + oldCount + " correct count = " + newCount + ") for type " + type + " for account ID " + accountId + " is fixed during resource count recalculation."); } diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index 7dca79e959b..856b607e657 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -896,7 +896,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe @Inject protected UserVmDao _userVmDao; @Inject - private ConfigurationDao _configDao; + protected ConfigurationDao _configDao; @Inject private ConfigurationGroupDao _configGroupDao; @Inject @@ -908,7 +908,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe @Inject private DiskOfferingDao _diskOfferingDao; @Inject - private DomainDao _domainDao; + protected DomainDao _domainDao; @Inject private AccountDao _accountDao; @Inject @@ -964,7 +964,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe @Inject private HostTagsDao _hostTagsDao; @Inject - private ConfigDepot _configDepot; + protected ConfigDepot _configDepot; @Inject private UserVmManager _userVmMgr; @Inject @@ -2196,7 +2196,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe final String groupName = cmd.getGroupName(); final String subGroupName = cmd.getSubGroupName(); final String parentName = cmd.getParentName(); - String scope = null; + ConfigKey.Scope scope = null; Long id = null; int paramCountCheck = 0; @@ -2212,35 +2212,35 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe } if (zoneId != null) { - scope = ConfigKey.Scope.Zone.toString(); + scope = ConfigKey.Scope.Zone; id = zoneId; paramCountCheck++; } if (clusterId != null) { - scope = ConfigKey.Scope.Cluster.toString(); + scope = ConfigKey.Scope.Cluster; id = clusterId; paramCountCheck++; } if (accountId != null) { Account account = _accountMgr.getAccount(accountId); _accountMgr.checkAccess(caller, null, false, account); - scope = ConfigKey.Scope.Account.toString(); + scope = ConfigKey.Scope.Account; id = accountId; paramCountCheck++; } if (domainId != null) { _accountMgr.checkAccess(caller, _domainDao.findById(domainId)); - scope = ConfigKey.Scope.Domain.toString(); + scope = ConfigKey.Scope.Domain; id = domainId; paramCountCheck++; } if (storagepoolId != null) { - scope = ConfigKey.Scope.StoragePool.toString(); + scope = ConfigKey.Scope.StoragePool; id = storagepoolId; paramCountCheck++; } if (imageStoreId != null) { - scope = ConfigKey.Scope.ImageStore.toString(); + scope = ConfigKey.Scope.ImageStore; id = imageStoreId; paramCountCheck++; } @@ -2295,19 +2295,19 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe // hidden configurations are not displayed using the search API sc.addAnd("category", SearchCriteria.Op.NEQ, "Hidden"); - if (scope != null && !scope.isEmpty()) { + if (scope != null) { // getting the list of parameters at requested scope if (ConfigurationManagerImpl.ENABLE_ACCOUNT_SETTINGS_FOR_DOMAIN.value() - && scope.equals(ConfigKey.Scope.Domain.toString())) { - sc.addAnd("scope", SearchCriteria.Op.IN, ConfigKey.Scope.Domain.toString(), ConfigKey.Scope.Account.toString()); + && scope.equals(ConfigKey.Scope.Domain)) { + sc.addAnd("scope", SearchCriteria.Op.BINARY_OR, (ConfigKey.Scope.Domain.getBitValue() | ConfigKey.Scope.Account.getBitValue())); } else { - sc.addAnd("scope", SearchCriteria.Op.EQ, scope); + sc.addAnd("scope", SearchCriteria.Op.BINARY_OR, scope.getBitValue()); } } final Pair, Integer> result = _configDao.searchAndCount(sc, searchFilter); - if (scope != null && !scope.isEmpty()) { + if (scope != null) { // Populate values corresponding the resource id final List configVOList = new ArrayList<>(); for (final ConfigurationVO param : result.first()) { @@ -2315,13 +2315,8 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe if (configVo != null) { final ConfigKey key = _configDepot.get(param.getName()); if (key != null) { - if (scope.equals(ConfigKey.Scope.Domain.toString())) { - Object value = key.valueInDomain(id); - configVo.setValue(value == null ? null : value.toString()); - } else { - Object value = key.valueIn(id); - configVo.setValue(value == null ? null : value.toString()); - } + Object value = key.valueInScope(scope, id); + configVo.setValue(value == null ? null : value.toString()); configVOList.add(configVo); } else { logger.warn("ConfigDepot could not find parameter " + param.getName() + " for scope " + scope); diff --git a/server/src/main/java/com/cloud/server/StatsCollector.java b/server/src/main/java/com/cloud/server/StatsCollector.java index 611b8c80cff..31d8c80329c 100644 --- a/server/src/main/java/com/cloud/server/StatsCollector.java +++ b/server/src/main/java/com/cloud/server/StatsCollector.java @@ -2011,7 +2011,7 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc Integer maxRetentionTime = vmStatsMaxRetentionTime.value(); if (maxRetentionTime <= 0) { logger.debug(String.format("Skipping VM stats cleanup. The [%s] parameter [%s] is set to 0 or less than 0.", - vmStatsMaxRetentionTime.scope(), vmStatsMaxRetentionTime.toString())); + ConfigKey.Scope.decodeAsCsv(vmStatsMaxRetentionTime.getScopeBitmask()), vmStatsMaxRetentionTime.toString())); return; } logger.trace("Removing older VM stats records."); @@ -2029,7 +2029,7 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc if (maxRetentionTime <= 0) { if (logger.isDebugEnabled()) { logger.debug(String.format("Skipping Volume stats cleanup. The [%s] parameter [%s] is set to 0 or less than 0.", - vmDiskStatsMaxRetentionTime.scope(), vmDiskStatsMaxRetentionTime.toString())); + ConfigKey.Scope.decodeAsCsv(vmDiskStatsMaxRetentionTime.getScopeBitmask()), vmDiskStatsMaxRetentionTime.toString())); } return; } diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index 03f24dde27d..938ceb2b092 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -3010,7 +3010,7 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C long totalSize = pool.getCapacityBytes(); long usedSize = getUsedSize(pool); double usedPercentage = ((double)usedSize / (double)totalSize); - double storageUsedThreshold = CapacityManager.StorageCapacityDisableThreshold.valueIn(pool.getDataCenterId()); + double storageUsedThreshold = CapacityManager.StorageCapacityDisableThreshold.valueIn(pool.getId()); if (logger.isDebugEnabled()) { logger.debug("Checking pool {} for storage, totalSize: {}, usedBytes: {}, usedPct: {}, disable threshold: {}", pool, pool.getCapacityBytes(), pool.getUsedBytes(), usedPercentage, storageUsedThreshold); } @@ -3282,7 +3282,7 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C logger.debug("Total capacity of the pool {} is {}", poolVO, toHumanReadableSize(totalOverProvCapacity)); - double storageAllocatedThreshold = CapacityManager.StorageAllocatedCapacityDisableThreshold.valueIn(pool.getDataCenterId()); + double storageAllocatedThreshold = CapacityManager.StorageAllocatedCapacityDisableThreshold.valueIn(pool.getId()); if (logger.isDebugEnabled()) { logger.debug("Checking pool: {} for storage allocation , maxSize : {}, " + @@ -3302,12 +3302,12 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C if (!forVolumeResize) { return false; } - if (!AllowVolumeReSizeBeyondAllocation.valueIn(pool.getDataCenterId())) { + if (!AllowVolumeReSizeBeyondAllocation.valueIn(pool.getId())) { logger.debug(String.format("Skipping the pool %s as %s is false", pool, AllowVolumeReSizeBeyondAllocation.key())); return false; } - double storageAllocatedThresholdForResize = CapacityManager.StorageAllocatedCapacityDisableThresholdForVolumeSize.valueIn(pool.getDataCenterId()); + double storageAllocatedThresholdForResize = CapacityManager.StorageAllocatedCapacityDisableThresholdForVolumeSize.valueIn(pool.getId()); if (usedPercentage > storageAllocatedThresholdForResize) { logger.debug(String.format("Skipping the pool %s since its allocated percentage: %s has crossed the allocated %s: %s", pool, usedPercentage, CapacityManager.StorageAllocatedCapacityDisableThresholdForVolumeSize.key(), storageAllocatedThresholdForResize)); @@ -3622,8 +3622,8 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C templateVO = _templateStoreDao.findByStoreTemplate(store.getId(), templateId); if (templateVO != null) { try { - if (SystemVmTemplateRegistration.validateIfSeeded( - url, templateVO.getInstallPath(), nfsVersion)) { + if (systemVmTemplateRegistration.validateIfSeeded( + templateVO, url, templateVO.getInstallPath(), nfsVersion)) { continue; } } catch (Exception e) { diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index a371a064701..12d3d315052 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -164,6 +164,7 @@ import com.cloud.resource.ResourceState; import com.cloud.serializer.GsonHelper; import com.cloud.server.ManagementService; import com.cloud.server.ResourceTag; +import com.cloud.server.StatsCollector; import com.cloud.server.TaggedResourceService; import com.cloud.service.ServiceOfferingVO; import com.cloud.service.dao.ServiceOfferingDao; @@ -353,9 +354,10 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic @Inject private BackupDao backupDao; @Inject + private StatsCollector statsCollector; + @Inject HostPodDao podDao; - protected Gson _gson; private static final List SupportedHypervisorsForVolResize = Arrays.asList(HypervisorType.KVM, HypervisorType.XenServer, @@ -2482,10 +2484,10 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic return existingVolumeOfVm; } - protected StoragePool getPoolForAllocatedOrUploadedVolumeForAttach(final VolumeInfo volumeToAttach, final UserVmVO vm) { + protected StoragePool getSuitablePoolForAllocatedOrUploadedVolumeForAttach(final VolumeInfo volumeToAttach, final UserVmVO vm) { DataCenter zone = _dcDao.findById(vm.getDataCenterId()); Pair clusterHostId = virtualMachineManager.findClusterAndHostIdForVm(vm, false); - long podId = vm.getPodIdToDeployIn(); + Long podId = vm.getPodIdToDeployIn(); if (clusterHostId.first() != null) { Cluster cluster = clusterDao.findById(clusterHostId.first()); podId = cluster.getPodId(); @@ -2497,12 +2499,8 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic offering.isUseLocalStorage(), offering.isRecreatable(), volumeToAttach.getTemplateId()); diskProfile.setHyperType(vm.getHypervisorType()); - StoragePool pool = _volumeMgr.findStoragePool(diskProfile, zone, pod, clusterHostId.first(), + return _volumeMgr.findStoragePool(diskProfile, zone, pod, clusterHostId.first(), clusterHostId.second(), vm, Collections.emptySet()); - if (pool == null) { - throw new CloudRuntimeException(String.format("Failed to find a primary storage for volume in state: %s", volumeToAttach.getState())); - } - return pool; } protected VolumeInfo createVolumeOnPrimaryForAttachIfNeeded(final VolumeInfo volumeToAttach, final UserVmVO vm, VolumeVO existingVolumeOfVm) { @@ -2520,7 +2518,13 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic } } if (destPrimaryStorage == null) { - destPrimaryStorage = getPoolForAllocatedOrUploadedVolumeForAttach(volumeToAttach, vm); + destPrimaryStorage = getSuitablePoolForAllocatedOrUploadedVolumeForAttach(volumeToAttach, vm); + if (destPrimaryStorage == null) { + if (Volume.State.Allocated.equals(volumeToAttach.getState()) && State.Stopped.equals(vm.getState())) { + return newVolumeOnPrimaryStorage; + } + throw new CloudRuntimeException(String.format("Failed to find a primary storage for volume in state: %s", volumeToAttach.getState())); + } } try { if (volumeOnSecondary && Storage.StoragePoolType.PowerFlex.equals(destPrimaryStorage.getPoolType())) { @@ -5204,6 +5208,21 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic return workJob; } + @Override + public Long getVolumePhysicalSize(ImageFormat format, String path, String chainInfo) { + VolumeStats vs = null; + if (format == ImageFormat.VHD || format == ImageFormat.QCOW2 || format == ImageFormat.RAW) { + if (path != null) { + vs = statsCollector.getVolumeStats(path); + } + } else if (format == ImageFormat.OVA) { + if (chainInfo != null) { + vs = statsCollector.getVolumeStats(chainInfo); + } + } + return (vs == null) ? null : vs.getPhysicalSize(); + } + @Override public String getConfigComponentName() { return VolumeApiService.class.getSimpleName(); diff --git a/server/src/main/java/com/cloud/user/AccountManagerImpl.java b/server/src/main/java/com/cloud/user/AccountManagerImpl.java index 1422f39f1fe..a3ae3fff01f 100644 --- a/server/src/main/java/com/cloud/user/AccountManagerImpl.java +++ b/server/src/main/java/com/cloud/user/AccountManagerImpl.java @@ -498,8 +498,8 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M if (apiNameList == null) { long startTime = System.nanoTime(); - apiNameList = new ArrayList(); - Set> cmdClasses = new LinkedHashSet>(); + apiNameList = new ArrayList<>(); + Set> cmdClasses = new LinkedHashSet<>(); for (PluggableService service : services) { logger.debug(String.format("getting api commands of service: %s", service.getClass().getName())); cmdClasses.addAll(service.getCommands()); @@ -513,7 +513,7 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M } protected List createApiNameList(Set> cmdClasses) { - List apiNameList = new ArrayList(); + List apiNameList = new ArrayList<>(); for (Class cmdClass : cmdClasses) { APICommand apiCmdAnnotation = cmdClass.getAnnotation(APICommand.class); @@ -698,7 +698,7 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M return; } - HashMap> domains = new HashMap>(); + HashMap> domains = new HashMap<>(); for (ControlledEntity entity : entities) { long domainId = entity.getDomainId(); @@ -713,7 +713,7 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M List toBeChecked = domains.get(entity.getDomainId()); // for templates, we don't have to do cross domains check if (toBeChecked == null) { - toBeChecked = new ArrayList(); + toBeChecked = new ArrayList<>(); domains.put(domainId, toBeChecked); } toBeChecked.add(entity); @@ -722,7 +722,11 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M for (SecurityChecker checker : _securityCheckers) { if (checker.checkAccess(caller, entity, accessType, apiName)) { if (logger.isDebugEnabled()) { - logger.debug("Access to " + entity + " granted to " + caller + " by " + checker.getName()); + User user = CallContext.current().getCallingUser(); + String userName = ""; + if (user != null) + userName = user.getUsername(); + logger.debug("Access to {} granted to {} by {} on behalf of user {}", entity, caller, checker.getName(), userName); } granted = true; break; @@ -1023,12 +1027,12 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M logger.debug("Deleting networks for account {}", account); List networks = _networkDao.listByOwner(accountId); if (networks != null) { - Collections.sort(networks, new Comparator() { + Collections.sort(networks, new Comparator<>() { @Override public int compare(NetworkVO network1, NetworkVO network2) { if (network1.getGuestType() != network2.getGuestType() && Network.GuestType.Isolated.equals(network2.getGuestType())) { return -1; - }; + } return 1; } }); @@ -1300,7 +1304,7 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M final String accountNameFinal = accountName; final Long domainIdFinal = domainId; final String accountUUIDFinal = accountUUID; - Pair pair = Transaction.execute(new TransactionCallback>() { + Pair pair = Transaction.execute(new TransactionCallback<>() { @Override public Pair doInTransaction(TransactionStatus status) { // create account @@ -1323,7 +1327,7 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M user.setRegistrationToken(registrationToken); } - return new Pair(user.getId(), account); + return new Pair<>(user.getId(), account); } }); @@ -1332,7 +1336,7 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M // create correct account and group association based on accountType if (accountType != Account.Type.PROJECT) { - Map accountGroupMap = new HashMap(); + Map accountGroupMap = new HashMap<>(); accountGroupMap.put(account.getId(), (long) (accountType.ordinal() + 1)); _messageBus.publish(_name, MESSAGE_ADD_ACCOUNT_EVENT, PublishScope.LOCAL, accountGroupMap); } @@ -1476,7 +1480,7 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M if (!_userAccountDao.validateUsernameInDomain(userName, domainId)) { throw new CloudRuntimeException(String.format("The user %s already exists in domain %s", userName, domain)); } - UserVO user = null; + UserVO user; user = createUser(account.getId(), userName, password, firstName, lastName, email, timeZone, userUUID, source); return user; } @@ -1731,7 +1735,7 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M Long callingUserId = CallContext.current().getCallingUserId(); Account callingAccount = CallContext.current().getCallingAccount(); ActionEventUtils.onActionEvent(callingUserId, callingAccount.getAccountId(), callingAccount.getDomainId(), - EventTypes.API_KEY_ACCESS_UPDATE, "Api key access was changed for the User to " + access.toString(), + EventTypes.API_KEY_ACCESS_UPDATE, "Api key access was changed for the User to " + access, user.getId(), ApiCommandResourceType.User.toString()); } catch (IllegalArgumentException ex) { throw new InvalidParameterValueException("ApiKeyAccess value can only be Enabled/Disabled/Inherit"); @@ -1747,7 +1751,7 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M Long callingUserId = CallContext.current().getCallingUserId(); Account callingAccount = CallContext.current().getCallingAccount(); ActionEventUtils.onActionEvent(callingUserId, callingAccount.getAccountId(), callingAccount.getDomainId(), - EventTypes.API_KEY_ACCESS_UPDATE, "Api key access was changed for the Account to " + access.toString(), + EventTypes.API_KEY_ACCESS_UPDATE, "Api key access was changed for the Account to " + access, account.getId(), ApiCommandResourceType.Account.toString()); } catch (IllegalArgumentException ex) { throw new InvalidParameterValueException("ApiKeyAccess value can only be Enabled/Disabled/Inherit"); @@ -1837,7 +1841,7 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M checkAccess(caller, AccessType.OperateEntry, true, account); - boolean success = Transaction.execute(new TransactionCallback() { + boolean success = Transaction.execute(new TransactionCallback<>() { @Override public Boolean doInTransaction(TransactionStatus status) { boolean success = doSetUserStatus(userId, State.ENABLED); @@ -1892,7 +1896,7 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M // make sure the account is enabled too // if the user is either locked already or disabled already, don't change state...only lock currently enabled // users - boolean success = true; + boolean success; if (user.getState().equals(State.LOCKED)) { // already locked...no-op return _userAccountDao.findById(userId); @@ -1995,7 +1999,7 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M public AccountVO enableAccount(String accountName, Long domainId, Long accountId) { // Check if account exists - Account account = null; + Account account; if (accountId != null) { account = _accountDao.findById(accountId); } else { @@ -2021,7 +2025,7 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M return _accountDao.findById(account.getId()); } else { - throw new CloudRuntimeException(String.format("Unable to enable account %s in domain %s", account, accountName, _domainMgr.getDomain(domainId))); + throw new CloudRuntimeException(String.format("Unable to enable account %s[%s] in domain %s", accountName, account.getUuid(), _domainMgr.getDomain(domainId))); } } @@ -2030,7 +2034,7 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M public AccountVO lockAccount(String accountName, Long domainId, Long accountId) { Account caller = getCurrentCallingAccount(); - Account account = null; + Account account; if (accountId != null) { account = _accountDao.findById(accountId); } else { @@ -2051,7 +2055,7 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M CallContext.current().putContextParameter(Account.class, account.getUuid()); return _accountDao.findById(account.getId()); } else { - throw new CloudRuntimeException(String.format("Unable to lock account %s by accountId: %d OR by name: %s in domain %d", account, accountId, accountName, _domainMgr.getDomain(domainId))); + throw new CloudRuntimeException(String.format("Unable to lock account %s by accountId: %d OR by name: %s in domain %s", account, accountId, accountName, _domainMgr.getDomain(domainId))); } } @@ -2060,7 +2064,7 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M public AccountVO disableAccount(String accountName, Long domainId, Long accountId) throws ConcurrentOperationException, ResourceUnavailableException { Account caller = getCurrentCallingAccount(); - Account account = null; + Account account; if (accountId != null) { account = _accountDao.findById(accountId); } else { @@ -2097,8 +2101,8 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M String networkDomain = cmd.getNetworkDomain(); final Map details = cmd.getDetails(); - boolean success = false; - Account account = null; + boolean success; + Account account; if (accountId != null) { account = _accountDao.findById(accountId); } else { @@ -2159,7 +2163,7 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M if (roleNotFound) { throw new InvalidParameterValueException(String.format("Role with ID '%s' is not " + "found or not available for the account '%s' in the domain '%s'.", - roleId.toString(), account, _domainMgr.getDomain(domainId))); + roleId, account, _domainMgr.getDomain(domainId))); } Role role = roleService.findRole(roleId); @@ -2244,7 +2248,7 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M return true; // no need to create a new user object for this user } - return Transaction.execute(new TransactionCallback() { + return Transaction.execute(new TransactionCallback<>() { @Override public Boolean doInTransaction(TransactionStatus status) { UserVO newUser = new UserVO(user); @@ -2560,7 +2564,7 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M } // Create the account - return Transaction.execute(new TransactionCallback() { + return Transaction.execute(new TransactionCallback<>() { @Override public AccountVO doInTransaction(TransactionStatus status) { AccountVO account = _accountDao.persist(new AccountVO(accountName, domainId, networkDomain, accountType, roleId, uuid)); @@ -2650,12 +2654,12 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M long tolerance = Long.parseLong(singleSignOnTolerance); String signature = null; long timestamp = 0L; - String unsignedRequest = null; + String unsignedRequest; StringBuffer unsignedRequestBuffer = new StringBuffer(); // - build a request string with sorted params, make sure it's all lowercase // - sign the request, verify the signature is the same - List parameterNames = new ArrayList(); + List parameterNames = new ArrayList<>(); for (Object paramNameObj : requestParameters.keySet()) { parameterNames.add((String)paramNameObj); // put the name in a list that we'll sort later @@ -2780,7 +2784,7 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M UserAccount userAccount = _userAccountDao.getUserAccount(username, domainId); boolean authenticated = false; - HashSet actionsOnFailedAuthenticaion = new HashSet(); + HashSet actionsOnFailedAuthenticaion = new HashSet<>(); User.Source userSource = userAccount != null ? userAccount.getSource() : User.Source.UNKNOWN; for (UserAuthenticator authenticator : _userAuthenticators) { final String[] secretCodeArray = (String[])requestParameters.get(ApiConstants.SECRET_CODE); @@ -2886,7 +2890,7 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M preventRootDomainAdminAccessToRootAdminKeys(caller, account); checkAccess(caller, account); - Map keys = new HashMap(); + Map keys = new HashMap<>(); keys.put("apikey", user.getApiKey()); keys.put("secretkey", user.getSecretKey()); @@ -2898,7 +2902,7 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M } } - return new Pair>(apiKeyAccess, keys); + return new Pair<>(apiKeyAccess, keys); } protected void preventRootDomainAdminAccessToRootAdminKeys(User caller, ControlledEntity account) { @@ -2999,8 +3003,8 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M try { UserVO updatedUser = _userDao.createForUpdate(); - String encodedKey = null; - Pair userAcct = null; + String encodedKey; + Pair userAcct; int retryLimit = 10; do { // FIXME: what algorithm should we use for API keys? @@ -3026,9 +3030,9 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M private String createUserSecretKey(long userId) { try { UserVO updatedUser = _userDao.createForUpdate(); - String encodedKey = null; + String encodedKey; int retryLimit = 10; - UserVO userBySecretKey = null; + UserVO userBySecretKey; do { KeyGenerator generator = KeyGenerator.getInstance("HmacSHA1"); SecretKey key = generator.generateKey(); @@ -3136,8 +3140,8 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M throw new InvalidParameterValueException("Account and projectId can't be specified together"); } - Account userAccount = null; - Domain domain = null; + Account userAccount; + Domain domain; if (domainId != null) { userAccount = _accountDao.findActiveAccount(accountName, domainId); domain = _domainDao.findById(domainId); @@ -3262,7 +3266,7 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M @Override public List listAclGroupsByAccount(Long accountId) { if (_querySelectors == null || _querySelectors.size() == 0) { - return new ArrayList(); + return new ArrayList<>(); } QuerySelector qs = _querySelectors.get(0); @@ -3522,7 +3526,7 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M } protected UserTwoFactorAuthenticationSetupResponse disableTwoFactorAuthentication(Long userId, Account caller, Account owner) { - UserVO userVO = null; + UserVO userVO; if (userId != null) { userVO = validateUser(userId); owner = _accountService.getActiveAccountById(userVO.getAccountId()); diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 6af1ad0c310..5ad8cf812fd 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -135,8 +135,8 @@ import org.apache.cloudstack.storage.template.VnfTemplateManager; import org.apache.cloudstack.userdata.UserDataManager; import org.apache.cloudstack.utils.bytescale.ByteScaleUtils; import org.apache.cloudstack.utils.security.ParserUtils; -import org.apache.cloudstack.vm.UnmanagedVMsManager; import org.apache.cloudstack.vm.schedule.VMScheduleManager; +import org.apache.cloudstack.vm.UnmanagedVMsManager; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.math.NumberUtils; @@ -2141,7 +2141,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir throw new InvalidParameterValueException("Unable to Scale VM, since disk offering strictness flag is not same for new service offering and old service offering"); } - if (currentServiceOffering.getDiskOfferingStrictness() && currentServiceOffering.getDiskOfferingId() != newServiceOffering.getDiskOfferingId()) { + if (currentServiceOffering.getDiskOfferingStrictness() && !currentServiceOffering.getDiskOfferingId().equals(newServiceOffering.getDiskOfferingId())) { throw new InvalidParameterValueException("Unable to Scale VM, since disk offering id associated with the old service offering is not same for new service offering"); } @@ -4527,7 +4527,11 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir } if (customParameters.containsKey(VmDetailConstants.ROOT_DISK_SIZE)) { - Long rootDiskSize = rootDiskSizeCustomParam * GiB_TO_BYTES; + Long rootDiskSize = NumbersUtil.parseLong(customParameters.get(VmDetailConstants.ROOT_DISK_SIZE), -1); + if (rootDiskSize <= 0) { + throw new InvalidParameterValueException("Root disk size should be a positive number."); + } + rootDiskSize = rootDiskSizeCustomParam * GiB_TO_BYTES; _volumeService.validateVolumeSizeInBytes(rootDiskSize); return rootDiskSize; } else { diff --git a/server/src/main/java/com/cloud/vm/snapshot/VMSnapshotManagerImpl.java b/server/src/main/java/com/cloud/vm/snapshot/VMSnapshotManagerImpl.java index 8d43875190f..907182edc2a 100644 --- a/server/src/main/java/com/cloud/vm/snapshot/VMSnapshotManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/snapshot/VMSnapshotManagerImpl.java @@ -27,6 +27,7 @@ import java.util.Map; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.storage.snapshot.SnapshotManager; import org.apache.cloudstack.annotation.AnnotationService; import org.apache.cloudstack.annotation.dao.AnnotationDao; import org.apache.cloudstack.api.ApiConstants; @@ -377,9 +378,14 @@ public class VMSnapshotManagerImpl extends MutualExclusiveIdsManagerBase impleme //StorageVMSnapshotStrategy - allows volume snapshots without memory; VM has to be in Running state; No limitation of the image format if the storage plugin supports volume snapshots; "kvm.vmstoragesnapshot.enabled" has to be enabled //Other Storage volume plugins could integrate this with their own functionality for group snapshots VMSnapshotStrategy snapshotStrategy = storageStrategyFactory.getVmSnapshotStrategy(userVmVo.getId(), rootVolumePool.getId(), snapshotMemory); - if (snapshotStrategy == null) { - String message = "KVM does not support the type of snapshot requested"; + String message; + if (!SnapshotManager.VmStorageSnapshotKvm.value() && !snapshotMemory) { + message = "Creating a snapshot of a running KVM instance without memory is not supported"; + } else { + message = "KVM does not support the type of snapshot requested"; + } + logger.debug(message); throw new CloudRuntimeException(message); } diff --git a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java index 2e52d1ccc44..816d8fd66c4 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java @@ -29,7 +29,13 @@ import java.util.TimerTask; import java.util.stream.Collectors; import com.amazonaws.util.CollectionUtils; +import com.cloud.alert.AlertManager; +import com.cloud.configuration.Resource; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.storage.Snapshot; import com.cloud.storage.VolumeApiService; +import com.cloud.user.DomainManager; +import com.cloud.user.ResourceLimitService; import com.cloud.utils.fsm.NoTransitionException; import com.cloud.vm.VirtualMachineManager; import javax.inject.Inject; @@ -37,6 +43,7 @@ import javax.naming.ConfigurationException; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.InternalIdentity; import org.apache.cloudstack.api.command.admin.backup.DeleteBackupOfferingCmd; import org.apache.cloudstack.api.command.admin.backup.ImportBackupOfferingCmd; import org.apache.cloudstack.api.command.admin.backup.ListBackupProviderOfferingsCmd; @@ -115,6 +122,7 @@ import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.Transaction; import com.cloud.utils.db.TransactionCallback; +import com.cloud.utils.db.TransactionCallbackNoReturn; import com.cloud.utils.db.TransactionLegacy; import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; @@ -139,6 +147,8 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { @Inject private AccountManager accountManager; @Inject + private DomainManager domainManager; + @Inject private VolumeDao volumeDao; @Inject private DataCenterDao dataCenterDao; @@ -164,6 +174,10 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { private VolumeApiService volumeApiService; @Inject private VolumeOrchestrationService volumeOrchestrationService; + @Inject + private ResourceLimitService resourceLimitMgr; + @Inject + private AlertManager alertManager; private AsyncJobDispatcher asyncJobDispatcher; private Timer backupTimer; @@ -396,8 +410,8 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_BACKUP_OFFERING_REMOVE, vm.getAccountId(), vm.getDataCenterId(), vm.getId(), "Backup-" + vm.getHostName() + "-" + vm.getUuid(), vm.getBackupOfferingId(), null, null, Backup.class.getSimpleName(), vm.getUuid()); - final BackupSchedule backupSchedule = backupScheduleDao.findByVM(vm.getId()); - if (backupSchedule != null) { + final List backupSchedules = backupScheduleDao.listByVM(vm.getId()); + for(BackupSchedule backupSchedule: backupSchedules) { backupScheduleDao.remove(backupSchedule.getId()); } result = true; @@ -415,6 +429,7 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { final DateUtil.IntervalType intervalType = cmd.getIntervalType(); final String scheduleString = cmd.getSchedule(); final TimeZone timeZone = TimeZone.getTimeZone(cmd.getTimezone()); + final Integer maxBackups = cmd.getMaxBackups(); if (intervalType == null) { throw new CloudRuntimeException("Invalid interval type provided"); @@ -427,12 +442,41 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { if (vm.getBackupOfferingId() == null) { throw new CloudRuntimeException("Cannot configure backup schedule for the VM without having any backup offering"); } + if (maxBackups != null && maxBackups <= 0) { + throw new InvalidParameterValueException(String.format("maxBackups [%s] for instance %s should be greater than 0.", maxBackups, vm.getName())); + } + + Backup.Type backupType = Backup.Type.valueOf(intervalType.name()); + int intervalMaxBackups = backupType.getMax(); + if (maxBackups != null && maxBackups > intervalMaxBackups) { + throw new InvalidParameterValueException(String.format("maxBackups [%s] for instance %s exceeds limit [%s] for interval type [%s].", maxBackups, vm.getName(), + intervalMaxBackups, intervalType)); + } + + Account owner = accountManager.getAccount(vm.getAccountId()); + + long accountLimit = resourceLimitMgr.findCorrectResourceLimitForAccount(owner, Resource.ResourceType.backup, null); + long domainLimit = resourceLimitMgr.findCorrectResourceLimitForDomain(domainManager.getDomain(owner.getDomainId()), Resource.ResourceType.backup, null); + if (maxBackups != null && !accountManager.isRootAdmin(owner.getId()) && ((accountLimit != -1 && maxBackups > accountLimit) || (domainLimit != -1 && maxBackups > domainLimit))) { + String message = "domain/account"; + if (owner.getType() == Account.Type.PROJECT) { + message = "domain/project"; + } + throw new InvalidParameterValueException("Max number of backups shouldn't exceed the " + message + " level backup limit"); + } final BackupOffering offering = backupOfferingDao.findById(vm.getBackupOfferingId()); if (offering == null || !offering.isUserDrivenBackupAllowed()) { throw new CloudRuntimeException("The selected backup offering does not allow user-defined backup schedule"); } + if (maxBackups == null && !"veeam".equals(offering.getProvider())) { + throw new CloudRuntimeException("Please specify the maximum number of buckets to retain."); + } + if (maxBackups != null && "veeam".equals(offering.getProvider())) { + throw new CloudRuntimeException("The maximum backups to retain cannot be configured through CloudStack for Veeam. Retention is managed directly in Veeam based on the settings specified when creating the backup job."); + } + final String timezoneId = timeZone.getID(); if (!timezoneId.equals(cmd.getTimezone())) { logger.warn("Using timezone: " + timezoneId + " for running this snapshot policy as an equivalent of " + cmd.getTimezone()); @@ -447,15 +491,16 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { final BackupScheduleVO schedule = backupScheduleDao.findByVMAndIntervalType(vmId, intervalType); if (schedule == null) { - return backupScheduleDao.persist(new BackupScheduleVO(vmId, intervalType, scheduleString, timezoneId, nextDateTime)); + return backupScheduleDao.persist(new BackupScheduleVO(vmId, intervalType, scheduleString, timezoneId, nextDateTime, maxBackups)); } schedule.setScheduleType((short) intervalType.ordinal()); schedule.setSchedule(scheduleString); schedule.setTimezone(timezoneId); schedule.setScheduledTimestamp(nextDateTime); + schedule.setMaxBackups(maxBackups); backupScheduleDao.update(schedule.getId(), schedule); - return backupScheduleDao.findByVM(vmId); + return backupScheduleDao.findById(schedule.getId()); } @Override @@ -469,7 +514,7 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { @Override @ActionEvent(eventType = EventTypes.EVENT_VM_BACKUP_SCHEDULE_DELETE, eventDescription = "deleting VM backup schedule") - public boolean deleteBackupSchedule(final Long vmId) { + public boolean deleteBackupSchedule(Long vmId) { final VMInstanceVO vm = findVmById(vmId); validateForZone(vm.getDataCenterId()); accountManager.checkAccess(CallContext.current().getCallingAccount(), null, true, vm); @@ -481,9 +526,30 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { return backupScheduleDao.remove(schedule.getId()); } + private void postCreateScheduledBackup(Backup.Type backupType, Long vmId) { + DateUtil.IntervalType intervalType = DateUtil.IntervalType.valueOf(backupType.name()); + final BackupScheduleVO schedule = backupScheduleDao.findByVMAndIntervalType(vmId, intervalType); + if (schedule == null) { + return; + } + Integer maxBackups = schedule.getMaxBackups(); + if (maxBackups == null) { + return; + } + List backups = backupDao.listBackupsByVMandIntervalType(vmId, backupType); + while (backups.size() > maxBackups) { + BackupVO oldestBackup = backups.get(0); + if (deleteBackup(oldestBackup.getId(), false)) { + ActionEventUtils.onCompletedActionEvent(User.UID_SYSTEM, oldestBackup.getAccountId(), EventVO.LEVEL_INFO, EventTypes.EVENT_VM_BACKUP_DELETE, + "Successfully deleted oldest backup: " + oldestBackup.getId(), oldestBackup.getId(), ApiCommandResourceType.Backup.toString(), 0); + } + backups.remove(oldestBackup); + } + } + @Override @ActionEvent(eventType = EventTypes.EVENT_VM_BACKUP_CREATE, eventDescription = "creating VM backup", async = true) - public boolean createBackup(final Long vmId) { + public boolean createBackup(final Long vmId, final Long scheduleId) throws ResourceAllocationException { final VMInstanceVO vm = findVmById(vmId); validateForZone(vm.getDataCenterId()); accountManager.checkAccess(CallContext.current().getCallingAccount(), null, true, vm); @@ -501,13 +567,65 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { throw new CloudRuntimeException("The assigned backup offering does not allow ad-hoc user backup"); } + Backup.Type type = getBackupType(scheduleId); + Account owner = accountManager.getAccount(vm.getAccountId()); + try { + resourceLimitMgr.checkResourceLimit(owner, Resource.ResourceType.backup); + } catch (ResourceAllocationException e) { + if (type != Backup.Type.MANUAL) { + String msg = "Backup resource limit exceeded for account id : " + owner.getId() + ". Failed to create backup"; + logger.warn(msg); + alertManager.sendAlert(AlertManager.AlertType.ALERT_TYPE_UPDATE_RESOURCE_COUNT, 0L, 0L, msg, "Backup resource limit exceeded for account id : " + owner.getId() + + ". Failed to create backups; please use updateResourceLimit to increase the limit"); + } + throw e; + } + + Long backupSize = 0L; + for (final Volume volume: volumeDao.findByInstance(vmId)) { + if (Volume.State.Ready.equals(volume.getState())) { + Long volumeSize = volumeApiService.getVolumePhysicalSize(volume.getFormat(), volume.getPath(), volume.getChainInfo()); + if (volumeSize == null) { + volumeSize = volume.getSize(); + } + backupSize += volumeSize; + } + } + try { + resourceLimitMgr.checkResourceLimit(owner, Resource.ResourceType.backup_storage, backupSize); + } catch (ResourceAllocationException e) { + if (type != Backup.Type.MANUAL) { + String msg = "Backup storage space resource limit exceeded for account id : " + owner.getId() + ". Failed to create backup"; + logger.warn(msg); + alertManager.sendAlert(AlertManager.AlertType.ALERT_TYPE_UPDATE_RESOURCE_COUNT, 0L, 0L, msg, "Backup storage space resource limit exceeded for account id : " + owner.getId() + + ". Failed to create backups; please use updateResourceLimit to increase the limit"); + } + throw e; + } + ActionEventUtils.onStartedActionEvent(User.UID_SYSTEM, vm.getAccountId(), EventTypes.EVENT_VM_BACKUP_CREATE, "creating backup for VM ID:" + vm.getUuid(), vmId, ApiCommandResourceType.VirtualMachine.toString(), true, 0); + final BackupProvider backupProvider = getBackupProvider(offering.getProvider()); - if (backupProvider != null && backupProvider.takeBackup(vm)) { + if (backupProvider != null) { + Pair result = backupProvider.takeBackup(vm); + if (!result.first()) { + throw new CloudRuntimeException("Failed to create VM backup"); + } + Backup backup = result.second(); + if (backup != null) { + BackupVO vmBackup = backupDao.findById(result.second().getId()); + vmBackup.setBackupIntervalType((short) type.ordinal()); + backupDao.update(vmBackup.getId(), vmBackup); + resourceLimitMgr.incrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup); + resourceLimitMgr.incrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup_storage, backup.getSize()); + } + if (type != Backup.Type.MANUAL) { + postCreateScheduledBackup(type, vm.getId()); + } return true; } throw new CloudRuntimeException("Failed to create VM backup"); @@ -682,6 +800,29 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { } } + private Backup.Type getBackupType(Long scheduleId) { + if (scheduleId.equals(Snapshot.MANUAL_POLICY_ID)) { + return Backup.Type.MANUAL; + } else { + BackupScheduleVO scheduleVO = backupScheduleDao.findById(scheduleId); + DateUtil.IntervalType intvType = scheduleVO.getScheduleType(); + return getBackupType(intvType); + } + } + + private Backup.Type getBackupType(DateUtil.IntervalType intvType) { + if (intvType.equals(DateUtil.IntervalType.HOURLY)) { + return Backup.Type.HOURLY; + } else if (intvType.equals(DateUtil.IntervalType.DAILY)) { + return Backup.Type.DAILY; + } else if (intvType.equals(DateUtil.IntervalType.WEEKLY)) { + return Backup.Type.WEEKLY; + } else if (intvType.equals(DateUtil.IntervalType.MONTHLY)) { + return Backup.Type.MONTHLY; + } + return null; + } + /** * Tries to update the state of given VM, given specified event * @param vm The VM to update its state @@ -858,6 +999,8 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { final BackupProvider backupProvider = getBackupProvider(offering.getProvider()); boolean result = backupProvider.deleteBackup(backup, forced); if (result) { + resourceLimitMgr.decrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup); + resourceLimitMgr.decrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup_storage, backup.getSize()); return backupDao.remove(backup.getId()); } throw new CloudRuntimeException("Failed to delete the backup"); @@ -925,6 +1068,10 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { public boolean configure(String name, Map params) throws ConfigurationException { super.configure(name, params); backgroundPollManager.submitTask(new BackupSyncTask(this)); + Backup.Type.HOURLY.setMax(BackupHourlyMax.value()); + Backup.Type.DAILY.setMax(BackupDailyMax.value()); + Backup.Type.WEEKLY.setMax(BackupWeeklyMax.value()); + Backup.Type.MONTHLY.setMax(BackupMonthlyMax.value()); return true; } @@ -1004,7 +1151,17 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { BackupFrameworkEnabled, BackupProviderPlugin, BackupSyncPollingInterval, - BackupEnableAttachDetachVolumes + BackupEnableAttachDetachVolumes, + BackupHourlyMax, + BackupDailyMax, + BackupWeeklyMax, + BackupMonthlyMax, + DefaultMaxAccountBackups, + DefaultMaxAccountBackupStorage, + DefaultMaxProjectBackups, + DefaultMaxProjectBackupStorage, + DefaultMaxDomainBackups, + DefaultMaxDomainBackupStorage }; } @@ -1137,6 +1294,7 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { true, 0); final Map params = new HashMap(); params.put(ApiConstants.VIRTUAL_MACHINE_ID, "" + vmId); + params.put(ApiConstants.SCHEDULE_ID, "" + backupScheduleId); params.put("ctxUserId", "1"); params.put("ctxAccountId", "" + vm.getAccountId()); params.put("ctxStartEventId", String.valueOf(eventId)); @@ -1205,7 +1363,7 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { * This background task syncs backups from providers side in CloudStack db * along with creation of usage records */ - private final class BackupSyncTask extends ManagedContextRunnable implements BackgroundPollTask { + protected final class BackupSyncTask extends ManagedContextRunnable implements BackgroundPollTask { private BackupManager backupManager; public BackupSyncTask(final BackupManager backupManager) { @@ -1253,13 +1411,81 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { } } + private Backup checkAndUpdateIfBackupEntryExistsForRestorePoint(Backup.RestorePoint restorePoint, List backupsInDb, VirtualMachine vm, Backup.Metric metric) { + for (final Backup backupInDb : backupsInDb) { + logger.debug(String.format("Checking if Backup %s with external ID %s for VM %s is valid", backupsInDb, backupInDb.getName(), vm)); + if (restorePoint.getId().equals(backupInDb.getExternalId())) { + logger.debug(String.format("Found Backup %s in both Database and Networker", backupInDb)); + if (metric != null) { + logger.debug(String.format("Update backup [%s] from [size: %s, protected size: %s] to [size: %s, protected size: %s].", + backupInDb, backupInDb.getSize(), backupInDb.getProtectedSize(), metric.getBackupSize(), metric.getDataSize())); + + resourceLimitMgr.decrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup_storage, backupInDb.getSize()); + ((BackupVO) backupInDb).setSize(metric.getBackupSize()); + ((BackupVO) backupInDb).setProtectedSize(metric.getDataSize()); + resourceLimitMgr.incrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup_storage, backupInDb.getSize()); + + backupDao.update(backupInDb.getId(), ((BackupVO) backupInDb)); + } + return backupInDb; + } + } + return null; + } + + private void syncBackups(BackupProvider backupProvider, VirtualMachine vm, Backup.Metric metric) { + Transaction.execute(new TransactionCallbackNoReturn() { + @Override + public void doInTransactionWithoutResult(TransactionStatus status) { + final List backupsInDb = backupDao.listByVmId(null, vm.getId()); + List restorePoints = backupProvider.listRestorePoints(vm); + if (restorePoints == null) { + return; + } + + final List removeList = backupsInDb.stream().map(InternalIdentity::getId).collect(Collectors.toList()); + for (final Backup.RestorePoint restorePoint : restorePoints) { + if (!(restorePoint.getId() == null || restorePoint.getType() == null || restorePoint.getCreated() == null)) { + Backup existingBackupEntry = checkAndUpdateIfBackupEntryExistsForRestorePoint(restorePoint, backupsInDb, vm, metric); + if (existingBackupEntry != null) { + removeList.remove(existingBackupEntry.getId()); + continue; + } + } + + Backup backup = backupProvider.createNewBackupEntryForRestorePoint(restorePoint, vm, metric); + if (backup != null) { + logger.warn("Added backup found in provider [" + backup + "]"); + resourceLimitMgr.incrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup); + resourceLimitMgr.incrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup_storage, backup.getSize()); + + logger.debug(String.format("Creating a new entry in backups: [id: %s, uuid: %s, vm_id: %s, external_id: %s, type: %s, date: %s, backup_offering_id: %s, account_id: %s, " + + "domain_id: %s, zone_id: %s].", backup.getId(), backup.getUuid(), backup.getVmId(), backup.getExternalId(), backup.getType(), backup.getDate(), + backup.getBackupOfferingId(), backup.getAccountId(), backup.getDomainId(), backup.getZoneId())); + + ActionEventUtils.onCompletedActionEvent(User.UID_SYSTEM, vm.getAccountId(), EventVO.LEVEL_INFO, EventTypes.EVENT_VM_BACKUP_CREATE, + String.format("Created backup %s for VM ID: %s", backup.getUuid(), vm.getUuid()), + vm.getId(), ApiCommandResourceType.VirtualMachine.toString(),0); + } + } + for (final Long backupIdToRemove : removeList) { + logger.warn(String.format("Removing backup with ID: [%s].", backupIdToRemove)); + Backup backup = backupDao.findById(backupIdToRemove); + resourceLimitMgr.decrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup); + resourceLimitMgr.decrementResourceCount(vm.getAccountId(), Resource.ResourceType.backup_storage, backup.getSize()); + backupDao.remove(backupIdToRemove); + } + } + }); + } + private void tryToSyncVMBackups(BackupProvider backupProvider, Map metrics, VirtualMachine vm) { try { final Backup.Metric metric = metrics.get(vm); if (metric != null) { logger.debug(String.format("Trying to sync backups of VM [%s] using backup provider [%s].", vm, backupProvider.getName())); // Sync out-of-band backups - backupProvider.syncBackups(vm, metric); + syncBackups(backupProvider, vm, metric); // Emit a usage event, update usage metric for the VM by the usage server UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_BACKUP_USAGE_METRIC, vm.getAccountId(), vm.getDataCenterId(), vm.getId(), "Backup-" + vm.getHostName() + "-" + vm.getUuid(), diff --git a/server/src/main/java/org/apache/cloudstack/storage/object/BucketApiServiceImpl.java b/server/src/main/java/org/apache/cloudstack/storage/object/BucketApiServiceImpl.java index 389ca52b03b..ca0e6291e52 100644 --- a/server/src/main/java/org/apache/cloudstack/storage/object/BucketApiServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/storage/object/BucketApiServiceImpl.java @@ -19,9 +19,12 @@ package org.apache.cloudstack.storage.object; import com.amazonaws.services.s3.internal.BucketNameUtils; import com.amazonaws.services.s3.model.IllegalBucketNameException; import com.cloud.agent.api.to.BucketTO; +import com.cloud.configuration.Resource; import com.cloud.event.ActionEvent; import com.cloud.event.EventTypes; import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.resourcelimit.ResourceLimitManagerImpl; import com.cloud.storage.BucketVO; import com.cloud.storage.DataStoreRole; import com.cloud.storage.dao.BucketDao; @@ -60,6 +63,8 @@ public class BucketApiServiceImpl extends ManagerBase implements BucketApiServic private BucketDao _bucketDao; @Inject private AccountManager _accountMgr; + @Inject + private ResourceLimitManagerImpl resourceLimitManager; @Inject private BucketStatisticsDao _bucketStatisticsDao; @@ -98,12 +103,18 @@ public class BucketApiServiceImpl extends ManagerBase implements BucketApiServic @Override public ConfigKey[] getConfigKeys() { return new ConfigKey[] { + DefaultMaxAccountBuckets, + DefaultMaxAccountObjectStorage, + DefaultMaxProjectBuckets, + DefaultMaxProjectObjectStorage, + DefaultMaxDomainBuckets, + DefaultMaxDomainObjectStorage }; } @Override @ActionEvent(eventType = EventTypes.EVENT_BUCKET_CREATE, eventDescription = "creating bucket", create = true) - public Bucket allocBucket(CreateBucketCmd cmd) { + public Bucket allocBucket(CreateBucketCmd cmd) throws ResourceAllocationException { try { BucketNameUtils.validateBucketName(cmd.getBucketName()); } catch (IllegalBucketNameException e) { @@ -125,6 +136,9 @@ public class BucketApiServiceImpl extends ManagerBase implements BucketApiServic return null; } + resourceLimitManager.checkResourceLimit(owner, Resource.ResourceType.bucket); + resourceLimitManager.checkResourceLimit(owner, Resource.ResourceType.object_storage, (cmd.getQuota() * Resource.ResourceType.bytesToGiB)); + BucketVO bucket = new BucketVO(ownerId, owner.getDomainId(), cmd.getObjectStoragePoolId(), cmd.getBucketName(), cmd.getQuota(), cmd.isVersioning(), cmd.isEncryption(), cmd.isObjectLocking(), cmd.getPolicy()); _bucketDao.persist(bucket); @@ -146,6 +160,7 @@ public class BucketApiServiceImpl extends ManagerBase implements BucketApiServic try { bucketTO = new BucketTO(objectStore.createBucket(bucket, objectLock)); bucketCreated = true; + resourceLimitManager.incrementResourceCount(bucket.getAccountId(), Resource.ResourceType.bucket); if (cmd.isVersioning()) { objectStore.setBucketVersioning(bucketTO); @@ -157,6 +172,7 @@ public class BucketApiServiceImpl extends ManagerBase implements BucketApiServic if (cmd.getQuota() != null) { objectStore.setQuota(bucketTO, cmd.getQuota()); + resourceLimitManager.incrementResourceCount(bucket.getAccountId(), Resource.ResourceType.object_storage, (cmd.getQuota() * Resource.ResourceType.bytesToGiB)); } if (cmd.getPolicy() != null) { @@ -188,6 +204,8 @@ public class BucketApiServiceImpl extends ManagerBase implements BucketApiServic ObjectStoreVO objectStoreVO = _objectStoreDao.findById(bucket.getObjectStoreId()); ObjectStoreEntity objectStore = (ObjectStoreEntity)_dataStoreMgr.getDataStore(objectStoreVO.getId(), DataStoreRole.Object); if (objectStore.deleteBucket(bucketTO)) { + resourceLimitManager.decrementResourceCount(bucket.getAccountId(), Resource.ResourceType.bucket); + resourceLimitManager.decrementResourceCount(bucket.getAccountId(), Resource.ResourceType.object_storage, (bucket.getQuota() * Resource.ResourceType.bytesToGiB)); return _bucketDao.remove(bucketId); } return false; @@ -195,7 +213,7 @@ public class BucketApiServiceImpl extends ManagerBase implements BucketApiServic @Override @ActionEvent(eventType = EventTypes.EVENT_BUCKET_UPDATE, eventDescription = "updating bucket") - public boolean updateBucket(UpdateBucketCmd cmd, Account caller) { + public boolean updateBucket(UpdateBucketCmd cmd, Account caller) throws ResourceAllocationException { BucketVO bucket = _bucketDao.findById(cmd.getId()); BucketTO bucketTO = new BucketTO(bucket); if (bucket == null) { @@ -204,6 +222,17 @@ public class BucketApiServiceImpl extends ManagerBase implements BucketApiServic _accountMgr.checkAccess(caller, null, true, bucket); ObjectStoreVO objectStoreVO = _objectStoreDao.findById(bucket.getObjectStoreId()); ObjectStoreEntity objectStore = (ObjectStoreEntity)_dataStoreMgr.getDataStore(objectStoreVO.getId(), DataStoreRole.Object); + Integer quota = cmd.getQuota(); + Integer quotaDelta = null; + + if (quota != null) { + quotaDelta = quota - bucket.getQuota(); + if (quotaDelta > 0) { + Account owner = _accountMgr.getActiveAccountById(bucket.getAccountId()); + resourceLimitManager.checkResourceLimit(owner, Resource.ResourceType.object_storage, (quotaDelta * Resource.ResourceType.bytesToGiB)); + } + } + try { if (cmd.getEncryption() != null) { if (cmd.getEncryption()) { @@ -231,6 +260,11 @@ public class BucketApiServiceImpl extends ManagerBase implements BucketApiServic if (cmd.getQuota() != null) { objectStore.setQuota(bucketTO, cmd.getQuota()); bucket.setQuota(cmd.getQuota()); + if (quotaDelta > 0) { + resourceLimitManager.incrementResourceCount(bucket.getAccountId(), Resource.ResourceType.object_storage, (quotaDelta * Resource.ResourceType.bytesToGiB)); + } else { + resourceLimitManager.decrementResourceCount(bucket.getAccountId(), Resource.ResourceType.object_storage, ((-quotaDelta) * Resource.ResourceType.bytesToGiB)); + } } _bucketDao.update(bucket.getId(), bucket); } catch (Exception e) { diff --git a/server/src/test/java/com/cloud/api/query/dao/SnapshotJoinDaoImplTest.java b/server/src/test/java/com/cloud/api/query/dao/SnapshotJoinDaoImplTest.java new file mode 100644 index 00000000000..c1159d3aa88 --- /dev/null +++ b/server/src/test/java/com/cloud/api/query/dao/SnapshotJoinDaoImplTest.java @@ -0,0 +1,109 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.api.query.dao; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.api.query.vo.SnapshotJoinVO; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; + +@RunWith(MockitoJUnitRunner.class) +public class SnapshotJoinDaoImplTest { + + @Spy + @InjectMocks + SnapshotJoinDaoImpl snapshotJoinDao = new SnapshotJoinDaoImpl(); + + @Mock + SearchCriteria mockSearchCriteria; + + @Before + public void setUp() { + SnapshotJoinVO mockSnap = mock(SnapshotJoinVO.class); + SearchBuilder mockSearchBuilder = mock(SearchBuilder.class); + when(mockSearchBuilder.entity()).thenReturn(mockSnap); + doReturn(mockSearchBuilder).when(snapshotJoinDao).createSearchBuilder(); + when(mockSearchBuilder.create()).thenReturn(mockSearchCriteria); + } + + @Test + public void testFindById_WithNullZoneId_EmptyResult() { + Long zoneId = null; + long id = 1L; + doReturn(Collections.emptyList()).when(snapshotJoinDao).search(mockSearchCriteria, null); + List result = snapshotJoinDao.findById(zoneId, id); + assertNull(result); + verify(mockSearchCriteria).setParameters("id", id); + verify(mockSearchCriteria, never()).setParameters("zoneId", zoneId); + } + + @Test + public void testFindById_WithValidZoneId_EmptyResult() { + Long zoneId = 1L; + long id = 1L; + doReturn(Collections.emptyList()).when(snapshotJoinDao).search(mockSearchCriteria, null); + List result = snapshotJoinDao.findById(zoneId, id); + assertNull(result); + verify(mockSearchCriteria).setParameters("id", id); + verify(mockSearchCriteria).setParameters("zoneId", zoneId); + } + + @Test + public void testFindById_WithValidResults() { + Long zoneId = 1L; + long id = 1L; + SnapshotJoinVO snapshot1 = mock(SnapshotJoinVO.class); + when(snapshot1.getSnapshotStorePair()).thenReturn("Primary_1"); + SnapshotJoinVO snapshot2 = mock(SnapshotJoinVO.class); + when(snapshot2.getSnapshotStorePair()).thenReturn("Image_1"); + doReturn(Arrays.asList(snapshot1, snapshot2)).when(snapshotJoinDao).search(mockSearchCriteria, null); + List result = snapshotJoinDao.findById(zoneId, id); + assertNotNull(result); + assertEquals(1, result.size()); + assertEquals("Image_1", result.get(0).getSnapshotStorePair()); + verify(mockSearchCriteria).setParameters("id", id); + verify(mockSearchCriteria).setParameters("zoneId", zoneId); + } + + @Test + public void testFindById_WithNullResults() { + long id = 1L; + doReturn(null).when(snapshotJoinDao).search(mockSearchCriteria, null); + List result = snapshotJoinDao.findById(null, id); + assertNull(result); + } +} diff --git a/server/src/test/java/com/cloud/configuration/ConfigurationManagerImplTest.java b/server/src/test/java/com/cloud/configuration/ConfigurationManagerImplTest.java index 0a045296821..8c714b57cdb 100644 --- a/server/src/test/java/com/cloud/configuration/ConfigurationManagerImplTest.java +++ b/server/src/test/java/com/cloud/configuration/ConfigurationManagerImplTest.java @@ -46,14 +46,17 @@ import com.cloud.storage.dao.VMTemplateZoneDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.user.Account; import com.cloud.user.User; +import com.cloud.utils.Pair; import com.cloud.utils.db.EntityManager; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.net.NetUtils; import com.cloud.vm.dao.VMInstanceDao; import org.apache.cloudstack.annotation.dao.AnnotationDao; +import org.apache.cloudstack.api.command.admin.config.ResetCfgCmd; import org.apache.cloudstack.api.command.admin.network.CreateNetworkOfferingCmd; import org.apache.cloudstack.api.command.admin.offering.UpdateDiskOfferingCmd; import org.apache.cloudstack.api.command.admin.zone.DeleteZoneCmd; +import org.apache.cloudstack.config.Configuration; import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; import org.apache.cloudstack.framework.config.ConfigDepot; import org.apache.cloudstack.framework.config.ConfigKey; @@ -61,6 +64,9 @@ import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.framework.config.impl.ConfigurationVO; import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; import org.apache.cloudstack.resourcedetail.DiskOfferingDetailVO; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.vm.UnmanagedVMsManager; import org.junit.Assert; import org.junit.Before; @@ -159,6 +165,10 @@ public class ConfigurationManagerImplTest { NetworkService networkService; @Mock NetworkModel networkModel; + @Mock + PrimaryDataStoreDao storagePoolDao; + @Mock + StoragePoolDetailsDao storagePoolDetailsDao; DeleteZoneCmd deleteZoneCmd; CreateNetworkOfferingCmd createNetworkOfferingCmd; @@ -175,7 +185,7 @@ public class ConfigurationManagerImplTest { @Before public void setUp() throws Exception { - Mockito.when(configurationVOMock.getScope()).thenReturn(ConfigKey.Scope.Global.name()); + Mockito.when(configurationVOMock.getScopes()).thenReturn(List.of(ConfigKey.Scope.Global)); Mockito.when(configDao.findByName(Mockito.anyString())).thenReturn(configurationVOMock); Mockito.when(configDepot.get(Mockito.anyString())).thenReturn(configKeyMock); @@ -442,6 +452,7 @@ public class ConfigurationManagerImplTest { Assert.assertNotNull(offering); } + @Test public void testValidateInvalidConfiguration() { Mockito.doReturn(null).when(configDao).findByName(Mockito.anyString()); String msg = configurationManagerImplSpy.validateConfigurationValue("test.config.name", "testvalue", ConfigKey.Scope.Global.toString()); @@ -451,7 +462,7 @@ public class ConfigurationManagerImplTest { @Test public void testValidateInvalidScopeForConfiguration() { ConfigurationVO cfg = mock(ConfigurationVO.class); - when(cfg.getScope()).thenReturn(ConfigKey.Scope.Account.toString()); + when(cfg.getScopes()).thenReturn(List.of(ConfigKey.Scope.Account)); Mockito.doReturn(cfg).when(configDao).findByName(Mockito.anyString()); String msg = configurationManagerImplSpy.validateConfigurationValue("test.config.name", "testvalue", ConfigKey.Scope.Domain.toString()); Assert.assertEquals("Invalid scope id provided for the parameter test.config.name", msg); @@ -460,12 +471,12 @@ public class ConfigurationManagerImplTest { @Test public void testValidateConfig_ThreadsOnKVMHostToTransferVMwareVMFiles_Failure() { ConfigurationVO cfg = mock(ConfigurationVO.class); - when(cfg.getScope()).thenReturn(ConfigKey.Scope.Global.toString()); + when(cfg.getScopes()).thenReturn(List.of(ConfigKey.Scope.Global)); ConfigKey configKey = UnmanagedVMsManager.ThreadsOnKVMHostToImportVMwareVMFiles; Mockito.doReturn(cfg).when(configDao).findByName(Mockito.anyString()); Mockito.doReturn(configKey).when(configurationManagerImplSpy._configDepot).get(configKey.key()); - String result = configurationManagerImplSpy.validateConfigurationValue(configKey.key(), "11", configKey.scope().toString()); + String result = configurationManagerImplSpy.validateConfigurationValue(configKey.key(), "11", configKey.getScopes().get(0).name()); Assert.assertNotNull(result); } @@ -473,24 +484,24 @@ public class ConfigurationManagerImplTest { @Test public void testValidateConfig_ThreadsOnKVMHostToTransferVMwareVMFiles_Success() { ConfigurationVO cfg = mock(ConfigurationVO.class); - when(cfg.getScope()).thenReturn(ConfigKey.Scope.Global.toString()); + when(cfg.getScopes()).thenReturn(List.of(ConfigKey.Scope.Global)); ConfigKey configKey = UnmanagedVMsManager.ThreadsOnKVMHostToImportVMwareVMFiles; Mockito.doReturn(cfg).when(configDao).findByName(Mockito.anyString()); Mockito.doReturn(configKey).when(configurationManagerImplSpy._configDepot).get(configKey.key()); - String msg = configurationManagerImplSpy.validateConfigurationValue(configKey.key(), "10", configKey.scope().toString()); + String msg = configurationManagerImplSpy.validateConfigurationValue(configKey.key(), "10", configKey.getScopes().get(0).name()); Assert.assertNull(msg); } @Test public void testValidateConfig_ConvertVmwareInstanceToKvmTimeout_Failure() { ConfigurationVO cfg = mock(ConfigurationVO.class); - when(cfg.getScope()).thenReturn(ConfigKey.Scope.Global.toString()); + when(cfg.getScopes()).thenReturn(List.of(ConfigKey.Scope.Global)); ConfigKey configKey = UnmanagedVMsManager.ConvertVmwareInstanceToKvmTimeout; Mockito.doReturn(cfg).when(configDao).findByName(Mockito.anyString()); Mockito.doReturn(configKey).when(configurationManagerImplSpy._configDepot).get(configKey.key()); configurationManagerImplSpy.populateConfigValuesForValidationSet(); - String result = configurationManagerImplSpy.validateConfigurationValue(configKey.key(), "0", configKey.scope().toString()); + String result = configurationManagerImplSpy.validateConfigurationValue(configKey.key(), "0", configKey.getScopes().get(0).name()); Assert.assertNotNull(result); } @@ -498,12 +509,12 @@ public class ConfigurationManagerImplTest { @Test public void testValidateConfig_ConvertVmwareInstanceToKvmTimeout_Success() { ConfigurationVO cfg = mock(ConfigurationVO.class); - when(cfg.getScope()).thenReturn(ConfigKey.Scope.Global.toString()); + when(cfg.getScopes()).thenReturn(List.of(ConfigKey.Scope.Global)); ConfigKey configKey = UnmanagedVMsManager.ConvertVmwareInstanceToKvmTimeout; Mockito.doReturn(cfg).when(configDao).findByName(Mockito.anyString()); Mockito.doReturn(configKey).when(configurationManagerImplSpy._configDepot).get(configKey.key()); configurationManagerImplSpy.populateConfigValuesForValidationSet(); - String msg = configurationManagerImplSpy.validateConfigurationValue(configKey.key(), "9", configKey.scope().toString()); + String msg = configurationManagerImplSpy.validateConfigurationValue(configKey.key(), "9", configKey.getScopes().get(0).name()); Assert.assertNull(msg); } @@ -851,4 +862,26 @@ public class ConfigurationManagerImplTest { boolean result = configurationManagerImplSpy.shouldValidateConfigRange(Config.ConsoleProxySessionMax.name(), "test", Config.ConsoleProxyUrlDomain); Assert.assertTrue(result); } + + @Test + public void testResetConfigurations() { + Long poolId = 1L; + ResetCfgCmd cmd = Mockito.mock(ResetCfgCmd.class); + Mockito.when(cmd.getCfgName()).thenReturn("pool.storage.capacity.disablethreshold"); + Mockito.when(cmd.getStoragepoolId()).thenReturn(poolId); + Mockito.when(cmd.getZoneId()).thenReturn(null); + Mockito.when(cmd.getClusterId()).thenReturn(null); + Mockito.when(cmd.getAccountId()).thenReturn(null); + Mockito.when(cmd.getDomainId()).thenReturn(null); + Mockito.when(cmd.getImageStoreId()).thenReturn(null); + + ConfigurationVO cfg = new ConfigurationVO("Advanced", "DEFAULT", "test", "pool.storage.capacity.disablethreshold", null, "description"); + cfg.setScope(10); + cfg.setDefaultValue(".85"); + Mockito.when(configDao.findByName("pool.storage.capacity.disablethreshold")).thenReturn(cfg); + Mockito.when(storagePoolDao.findById(poolId)).thenReturn(Mockito.mock(StoragePoolVO.class)); + + Pair result = configurationManagerImplSpy.resetConfiguration(cmd); + Assert.assertEquals(".85", result.second()); + } } diff --git a/server/src/test/java/com/cloud/resourcelimit/ResourceLimitManagerImplTest.java b/server/src/test/java/com/cloud/resourcelimit/ResourceLimitManagerImplTest.java index 34030626d22..34e4632d24a 100644 --- a/server/src/test/java/com/cloud/resourcelimit/ResourceLimitManagerImplTest.java +++ b/server/src/test/java/com/cloud/resourcelimit/ResourceLimitManagerImplTest.java @@ -23,9 +23,15 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import com.cloud.event.ActionEventUtils; +import com.cloud.event.EventTypes; +import com.cloud.utils.db.EntityManager; + +import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.response.AccountResponse; import org.apache.cloudstack.api.response.DomainResponse; import org.apache.cloudstack.api.response.TaggedResourceLimitAndCountResponse; +import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.reservation.dao.ReservationDao; import org.apache.commons.collections.CollectionUtils; @@ -39,6 +45,7 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InjectMocks; import org.mockito.Mock; +import org.mockito.MockedStatic; import org.mockito.Mockito; import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; @@ -67,6 +74,7 @@ import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.template.VirtualMachineTemplate; import com.cloud.user.Account; +import com.cloud.user.User; import com.cloud.user.AccountManager; import com.cloud.user.AccountVO; import com.cloud.user.ResourceLimitService; @@ -80,6 +88,9 @@ import com.cloud.vpc.MockResourceLimitManagerImpl; import junit.framework.TestCase; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + @RunWith(MockitoJUnitRunner.class) public class ResourceLimitManagerImplTest extends TestCase { private Logger logger = LogManager.getLogger(ResourceLimitManagerImplTest.class); @@ -118,7 +129,10 @@ public class ResourceLimitManagerImplTest extends TestCase { VolumeDao volumeDao; @Mock UserVmDao userVmDao; + @Mock + EntityManager entityManager; + private CallContext callContext; private List hostTags = List.of("htag1", "htag2", "htag3"); private List storageTags = List.of("stag1", "stag2"); @@ -136,10 +150,15 @@ public class ResourceLimitManagerImplTest extends TestCase { } catch (IllegalAccessException | NoSuchFieldException e) { logger.error("Failed to update configurations"); } + + Account account = mock(Account.class); + User user = mock(User.class); + CallContext.register(user, account); } @After public void tearDown() throws Exception { + CallContext.unregister(); } @Test @@ -415,6 +434,9 @@ public class ResourceLimitManagerImplTest extends TestCase { Mockito.when(resourceLimitDao.findByOwnerIdAndTypeAndTag(1L, Resource.ResourceOwnerType.Account, Resource.ResourceType.cpu, hostTags.get(0))).thenReturn(null); result = resourceLimitManager.findCorrectResourceLimitForAccount(account, Resource.ResourceType.cpu, hostTags.get(0)); Assert.assertEquals(defaultAccountCpuMax, result); + + result = resourceLimitManager.findCorrectResourceLimitForAccount(account, Resource.ResourceType.cpu, hostTags.get(0)); + Assert.assertEquals(defaultAccountCpuMax, result); } @Test @@ -449,24 +471,26 @@ public class ResourceLimitManagerImplTest extends TestCase { @Test public void testFindCorrectResourceLimitForAccountId1() { -// long accountId = 1L; -// Mockito.when(accountManager.isRootAdmin(accountId)).thenReturn(true); -// long result = resourceLimitManager.findCorrectResourceLimitForAccount(accountId, null, Resource.ResourceType.cpu); -// Assert.assertEquals(Resource.RESOURCE_UNLIMITED, result); -// -// accountId = 2L; -// Mockito.when(accountManager.isRootAdmin(accountId)).thenReturn(false); -// Long limit = 100L; -// long result = resourceLimitManager.findCorrectResourceLimitForAccount(accountId, limit, Resource.ResourceType.cpu); -// Assert.assertEquals(limit.longValue(), result); -// -// long defaultAccountCpuMax = 25L; -// Mockito.when(accountManager.isRootAdmin(accountId)).thenReturn(false); -// Map accountResourceLimitMap = new HashMap<>(); -// accountResourceLimitMap.put(Resource.ResourceType.cpu.name(), defaultAccountCpuMax); -// resourceLimitManager.accountResourceLimitMap = accountResourceLimitMap; -// result = resourceLimitManager.findCorrectResourceLimitForAccount(accountId, null, Resource.ResourceType.cpu); -// Assert.assertEquals(defaultAccountCpuMax, result); + long accountId = 1L; + Mockito.when(accountManager.isRootAdmin(accountId)).thenReturn(true); + long result = resourceLimitManager.findCorrectResourceLimitForAccount(accountId, null, Resource.ResourceType.cpu); + Assert.assertEquals(Resource.RESOURCE_UNLIMITED, result); + + accountId = 2L; + AccountVO account = mock(AccountVO.class); + Mockito.when(accountManager.isRootAdmin(accountId)).thenReturn(false); + Mockito.when(accountDao.findById(accountId)).thenReturn(account); + Long limit = 100L; + result = resourceLimitManager.findCorrectResourceLimitForAccount(accountId, limit, Resource.ResourceType.cpu); + Assert.assertEquals(limit.longValue(), result); + + long defaultAccountCpuMax = 25L; + Mockito.when(accountManager.isRootAdmin(accountId)).thenReturn(false); + Map accountResourceLimitMap = new HashMap<>(); + accountResourceLimitMap.put(Resource.ResourceType.cpu.name(), defaultAccountCpuMax); + resourceLimitManager.accountResourceLimitMap = accountResourceLimitMap; + result = resourceLimitManager.findCorrectResourceLimitForAccount(accountId, null, Resource.ResourceType.cpu); + Assert.assertEquals(defaultAccountCpuMax, result); } @Test @@ -1223,4 +1247,65 @@ public class ResourceLimitManagerImplTest extends TestCase { Mockito.verify(resourceLimitManager, Mockito.times(1)) .decrementResourceCountWithTag(accountId, Resource.ResourceType.memory, tag, Long.valueOf(memory)); } + + @Test + public void testUpdateResourceLimitForAccount() { + Long accountId = 1L; + Long resourceLimitId = 3L; + Integer typeId = 13; + Long maxGB = 10L; + Long maxBytes = maxGB * Resource.ResourceType.bytesToGiB; + + Account account = mock(Account.class); + when(entityManager.findById(Account.class, accountId)).thenReturn(account); + ResourceLimitVO resourceLimitVO = mock(ResourceLimitVO.class); + when(resourceLimitVO.getId()).thenReturn(resourceLimitId); + when(resourceLimitDao.findByOwnerIdAndTypeAndTag(accountId, Resource.ResourceOwnerType.Account, Resource.ResourceType.backup_storage, null)).thenReturn(resourceLimitVO); + + try (MockedStatic actionEventUtilsMockedStatic = Mockito.mockStatic(ActionEventUtils.class)) { + Mockito.when(ActionEventUtils.onActionEvent(Mockito.anyLong(), Mockito.anyLong(), + Mockito.anyLong(), + Mockito.anyString(), Mockito.anyString(), + Mockito.anyLong(), Mockito.anyString())).thenReturn(1L); + + resourceLimitManager.updateResourceLimit(accountId, null, typeId, maxGB, null); + + Mockito.verify(resourceLimitDao, Mockito.times(1)).update(resourceLimitId, maxBytes); + Mockito.verify(resourceLimitDao, Mockito.never()).persist(Mockito.any()); + actionEventUtilsMockedStatic.verify(() -> ActionEventUtils.onActionEvent(0L, 0L, 0L, EventTypes.EVENT_RESOURCE_LIMIT_UPDATE, + "Resource limit updated. Resource Type: " + Resource.ResourceType.backup_storage.toString() + ", New Value: " + maxBytes, + accountId, ApiCommandResourceType.Account.toString())); + } + } + + @Test + public void testUpdateResourceLimitForDomain() { + Long domainId = 2L; + Long resourceLimitId = 3L; + Integer typeId = 13; + Long maxGB = 10L; + Long maxBytes = maxGB * Resource.ResourceType.bytesToGiB; + + Domain domain = mock(Domain.class); + when(domain.getParent()).thenReturn(null); + when(entityManager.findById(Domain.class, domainId)).thenReturn(domain); + ResourceLimitVO resourceLimitVO = mock(ResourceLimitVO.class); + when(resourceLimitVO.getId()).thenReturn(resourceLimitId); + when(resourceLimitDao.findByOwnerIdAndTypeAndTag(domainId, Resource.ResourceOwnerType.Domain, Resource.ResourceType.backup_storage, null)).thenReturn(resourceLimitVO); + + try (MockedStatic actionEventUtilsMockedStatic = Mockito.mockStatic(ActionEventUtils.class)) { + Mockito.when(ActionEventUtils.onActionEvent(Mockito.anyLong(), Mockito.anyLong(), + Mockito.anyLong(), + Mockito.anyString(), Mockito.anyString(), + Mockito.anyLong(), Mockito.anyString())).thenReturn(1L); + + resourceLimitManager.updateResourceLimit(null, domainId, typeId, maxGB, null); + + Mockito.verify(resourceLimitDao, Mockito.times(1)).update(resourceLimitId, maxBytes); + Mockito.verify(resourceLimitDao, Mockito.never()).persist(Mockito.any()); + actionEventUtilsMockedStatic.verify(() -> ActionEventUtils.onActionEvent(0L, 0L, 0L, EventTypes.EVENT_RESOURCE_LIMIT_UPDATE, + "Resource limit updated. Resource Type: " + Resource.ResourceType.backup_storage.toString() + ", New Value: " + maxBytes, + domainId, ApiCommandResourceType.Domain.toString())); + } + } } diff --git a/server/src/test/java/com/cloud/server/ManagementServerImplTest.java b/server/src/test/java/com/cloud/server/ManagementServerImplTest.java index df94a7d2157..fbaea758f37 100644 --- a/server/src/test/java/com/cloud/server/ManagementServerImplTest.java +++ b/server/src/test/java/com/cloud/server/ManagementServerImplTest.java @@ -17,6 +17,7 @@ package com.cloud.server; import com.cloud.dc.Vlan.VlanType; +import com.cloud.domain.dao.DomainDao; import com.cloud.exception.InvalidParameterValueException; import com.cloud.host.DetailVO; import com.cloud.host.Host; @@ -48,15 +49,20 @@ import com.cloud.vm.dao.UserVmDetailsDao; import org.apache.cloudstack.annotation.dao.AnnotationDao; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.command.admin.config.ListCfgsByCmd; import org.apache.cloudstack.api.command.user.address.ListPublicIpAddressesCmd; import org.apache.cloudstack.api.command.user.ssh.RegisterSSHKeyPairCmd; import org.apache.cloudstack.api.command.user.userdata.DeleteUserDataCmd; import org.apache.cloudstack.api.command.user.userdata.ListUserDataCmd; import org.apache.cloudstack.api.command.user.userdata.RegisterUserDataCmd; +import org.apache.cloudstack.config.Configuration; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver; +import org.apache.cloudstack.framework.config.ConfigDepot; import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.framework.config.impl.ConfigurationVO; import org.apache.cloudstack.userdata.UserDataManager; import org.junit.After; import org.junit.Assert; @@ -131,6 +137,16 @@ public class ManagementServerImplTest { @Mock HostDetailsDao hostDetailsDao; + + @Mock + ConfigurationDao configDao; + + @Mock + ConfigDepot configDepot; + + @Mock + DomainDao domainDao; + private AutoCloseable closeable; @Before @@ -145,6 +161,9 @@ public class ManagementServerImplTest { spy._UserVmDetailsDao = userVmDetailsDao; spy._detailsDao = hostDetailsDao; spy.userDataManager = userDataManager; + spy._configDao = configDao; + spy._configDepot = configDepot; + spy._domainDao = domainDao; } @After @@ -684,4 +703,41 @@ public class ManagementServerImplTest { Mockito.when(driver.zoneWideVolumesAvailableWithoutClusterMotion()).thenReturn(false); Assert.assertTrue(spy.zoneWideVolumeRequiresStorageMotion(dataStore, host1, host2)); } + + @Test(expected = InvalidParameterValueException.class) + public void testSearchForConfigurationsMultipleIds() { + ListCfgsByCmd cmd = Mockito.mock(ListCfgsByCmd.class); + Mockito.when(cmd.getConfigName()).thenReturn("pool.storage.capacity.disablethreshold"); + Mockito.when(cmd.getZoneId()).thenReturn(1L); + Mockito.when(cmd.getStoragepoolId()).thenReturn(2L); + spy.searchForConfigurations(cmd); + } + + @Test + public void testSearchForConfigurations() { + Long poolId = 1L; + ListCfgsByCmd cmd = Mockito.mock(ListCfgsByCmd.class); + Mockito.when(cmd.getConfigName()).thenReturn("pool.storage.capacity.disablethreshold"); + Mockito.when(cmd.getStoragepoolId()).thenReturn(poolId); + Mockito.when(cmd.getZoneId()).thenReturn(null); + Mockito.when(cmd.getClusterId()).thenReturn(null); + Mockito.when(cmd.getAccountId()).thenReturn(null); + Mockito.when(cmd.getDomainId()).thenReturn(null); + Mockito.when(cmd.getImageStoreId()).thenReturn(null); + + SearchCriteria sc = Mockito.mock(SearchCriteria.class); + Mockito.when(configDao.createSearchCriteria()).thenReturn(sc); + ConfigurationVO cfg = new ConfigurationVO("Advanced", "DEFAULT", "test", "pool.storage.capacity.disablethreshold", null, "description"); + Mockito.when(configDao.searchAndCount(any(), any())).thenReturn(new Pair<>(List.of(cfg), 1)); + Mockito.when(configDao.findByName("pool.storage.capacity.disablethreshold")).thenReturn(cfg); + + ConfigKey storageDisableThreshold = new ConfigKey<>(ConfigKey.CATEGORY_ALERT, Double.class, "pool.storage.capacity.disablethreshold", "0.85", + "Percentage (as a value between 0 and 1) of storage utilization above which allocators will disable using the pool for low storage available.", + true, List.of(ConfigKey.Scope.StoragePool, ConfigKey.Scope.Zone)); + when(configDepot.get("pool.storage.capacity.disablethreshold")).thenReturn(storageDisableThreshold); + + Pair, Integer> result = spy.searchForConfigurations(cmd); + + Assert.assertEquals("0.85", result.first().get(0).getValue()); + } } diff --git a/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java b/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java index 4a28e044d9c..999bf85907b 100644 --- a/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java +++ b/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java @@ -780,7 +780,6 @@ public class StorageManagerImplTest { Mockito.when(pool.getId()).thenReturn(poolId); Mockito.when(pool.getCapacityBytes()).thenReturn(capacityBytes); - Mockito.when(pool.getDataCenterId()).thenReturn(zoneId); Mockito.when(storagePoolDao.findById(poolId)).thenReturn(pool); Mockito.when(pool.getPoolType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem); diff --git a/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java b/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java index 23b79e4ab03..a55f9b26a2c 100644 --- a/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java +++ b/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java @@ -2038,13 +2038,13 @@ public class VolumeApiServiceImplTest { when(volumeToAttach.getDiskOfferingId()).thenReturn(1L); when(volumeOrchestrationService.findStoragePool(any(DiskProfile.class), eq(zone), eq(pod), eq(1L), eq(2L), eq(vm), eq(Collections.emptySet()))) .thenReturn(pool); - StoragePool result = volumeApiServiceImpl.getPoolForAllocatedOrUploadedVolumeForAttach(volumeToAttach, vm); + StoragePool result = volumeApiServiceImpl.getSuitablePoolForAllocatedOrUploadedVolumeForAttach(volumeToAttach, vm); Assert.assertNotNull(result); Assert.assertEquals(pool, result); } - @Test(expected = CloudRuntimeException.class) - public void testGetPoolForAllocatedOrUploadedVolumeForAttach_NoPoolFound_ThrowsException() { + @Test + public void testGetPoolForAllocatedOrUploadedVolumeForAttach_NoSuitablePoolFound_ReturnsNull() { VolumeInfo volumeToAttach = Mockito.mock(VolumeInfo.class); UserVmVO vm = Mockito.mock(UserVmVO.class); DataCenterVO zone = mockZone(); @@ -2059,11 +2059,11 @@ public class VolumeApiServiceImplTest { when(volumeToAttach.getDiskOfferingId()).thenReturn(1L); when(volumeOrchestrationService.findStoragePool(any(DiskProfile.class), eq(zone), eq(pod), eq(1L), eq(2L), eq(vm), eq(Collections.emptySet()))) .thenReturn(null); - volumeApiServiceImpl.getPoolForAllocatedOrUploadedVolumeForAttach(volumeToAttach, vm); + Assert.assertNull(volumeApiServiceImpl.getSuitablePoolForAllocatedOrUploadedVolumeForAttach(volumeToAttach, vm)); } @Test - public void testGetPoolForAllocatedOrUploadedVolumeForAttach_NoCluster() { + public void testGetSuitablePoolForAllocatedOrUploadedVolumeForAttach_NoCluster() { VolumeInfo volumeToAttach = Mockito.mock(VolumeInfo.class); UserVmVO vm = Mockito.mock(UserVmVO.class); DataCenterVO zone = mockZone(); @@ -2077,7 +2077,7 @@ public class VolumeApiServiceImplTest { when(volumeToAttach.getDiskOfferingId()).thenReturn(1L); when(volumeOrchestrationService.findStoragePool(any(DiskProfile.class), eq(zone), eq(pod), eq(null), eq(2L), eq(vm), eq(Collections.emptySet()))) .thenReturn(pool); - StoragePool result = volumeApiServiceImpl.getPoolForAllocatedOrUploadedVolumeForAttach(volumeToAttach, vm); + StoragePool result = volumeApiServiceImpl.getSuitablePoolForAllocatedOrUploadedVolumeForAttach(volumeToAttach, vm); Assert.assertNotNull(result); Assert.assertEquals(pool, result); } @@ -2123,7 +2123,7 @@ public class VolumeApiServiceImplTest { UserVmVO vm = Mockito.mock(UserVmVO.class); StoragePool destPrimaryStorage = Mockito.mock(StoragePool.class); Mockito.doReturn(destPrimaryStorage).when(volumeApiServiceImpl) - .getPoolForAllocatedOrUploadedVolumeForAttach(volumeToAttach, vm); + .getSuitablePoolForAllocatedOrUploadedVolumeForAttach(volumeToAttach, vm); VolumeInfo newVolumeOnPrimaryStorage = Mockito.mock(VolumeInfo.class); try { Mockito.when(volumeOrchestrationService.createVolumeOnPrimaryStorage( @@ -2134,7 +2134,7 @@ public class VolumeApiServiceImplTest { } VolumeInfo result = volumeApiServiceImpl.createVolumeOnPrimaryForAttachIfNeeded(volumeToAttach, vm, null); Assert.assertSame(newVolumeOnPrimaryStorage, result); - verify(volumeApiServiceImpl).getPoolForAllocatedOrUploadedVolumeForAttach(volumeToAttach, vm); + verify(volumeApiServiceImpl).getSuitablePoolForAllocatedOrUploadedVolumeForAttach(volumeToAttach, vm); } @Test(expected = InvalidParameterValueException.class) @@ -2145,7 +2145,7 @@ public class VolumeApiServiceImplTest { StoragePool destPrimaryStorage = Mockito.mock(StoragePool.class); when(destPrimaryStorage.getPoolType()).thenReturn(Storage.StoragePoolType.PowerFlex); Mockito.doReturn(destPrimaryStorage).when(volumeApiServiceImpl) - .getPoolForAllocatedOrUploadedVolumeForAttach(volumeToAttach, vm); + .getSuitablePoolForAllocatedOrUploadedVolumeForAttach(volumeToAttach, vm); volumeApiServiceImpl.createVolumeOnPrimaryForAttachIfNeeded(volumeToAttach, vm, null); } @@ -2157,7 +2157,7 @@ public class VolumeApiServiceImplTest { StoragePool destPrimaryStorage = Mockito.mock(StoragePool.class); Mockito.when(destPrimaryStorage.getPoolType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem); Mockito.doReturn(destPrimaryStorage).when(volumeApiServiceImpl) - .getPoolForAllocatedOrUploadedVolumeForAttach(volumeToAttach, vm); + .getSuitablePoolForAllocatedOrUploadedVolumeForAttach(volumeToAttach, vm); try { Mockito.when(volumeOrchestrationService.createVolumeOnPrimaryStorage(vm, volumeToAttach, vm.getHypervisorType(), destPrimaryStorage)) .thenThrow(new NoTransitionException("Mocked exception")); @@ -2169,4 +2169,35 @@ public class VolumeApiServiceImplTest { ); Assert.assertTrue(exception.getMessage().contains("Failed to create volume on primary storage")); } + + @Test + public void testCreateVolumeOnSecondaryForAttachIfNeeded_NoSuitablePool_ThrowsException() { + VolumeInfo volumeToAttach = Mockito.mock(VolumeInfo.class); + Mockito.when(volumeToAttach.getState()).thenReturn(Volume.State.Uploaded); + UserVmVO vm = Mockito.mock(UserVmVO.class); + Mockito.doReturn(null).when(volumeApiServiceImpl) + .getSuitablePoolForAllocatedOrUploadedVolumeForAttach(volumeToAttach, vm); + CloudRuntimeException exception = Assert.assertThrows(CloudRuntimeException.class, () -> + volumeApiServiceImpl.createVolumeOnPrimaryForAttachIfNeeded(volumeToAttach, vm, null) + ); + Assert.assertTrue(exception.getMessage().startsWith("Failed to find a primary storage for volume")); + } + + @Test + public void testCreateVolumeOnSecondaryForAttachIfNeeded_NoSuitablePool_ReturnSameVolumeInfo() { + VolumeInfo volumeToAttach = Mockito.mock(VolumeInfo.class); + Mockito.when(volumeToAttach.getState()).thenReturn(Volume.State.Allocated); + UserVmVO vm = Mockito.mock(UserVmVO.class); + Mockito.when(vm.getState()).thenReturn(State.Stopped); + Mockito.doReturn(null).when(volumeApiServiceImpl) + .getSuitablePoolForAllocatedOrUploadedVolumeForAttach(volumeToAttach, vm); + VolumeInfo result = volumeApiServiceImpl.createVolumeOnPrimaryForAttachIfNeeded(volumeToAttach, vm, null); + Assert.assertSame(volumeToAttach, result); + try { + Mockito.verify(volumeOrchestrationService, Mockito.never()).createVolumeOnPrimaryStorage(Mockito.any(), + Mockito.any(), Mockito.any(), Mockito.any()); + } catch (NoTransitionException e) { + Assert.fail(); + } + } } diff --git a/server/src/test/java/com/cloud/vpc/dao/MockConfigurationDaoImpl.java b/server/src/test/java/com/cloud/vpc/dao/MockConfigurationDaoImpl.java index 012ae739bc7..90373724724 100644 --- a/server/src/test/java/com/cloud/vpc/dao/MockConfigurationDaoImpl.java +++ b/server/src/test/java/com/cloud/vpc/dao/MockConfigurationDaoImpl.java @@ -16,12 +16,14 @@ // under the License. package com.cloud.vpc.dao; -import com.cloud.utils.db.GenericDaoBase; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.framework.config.impl.ConfigurationVO; -import java.util.HashMap; -import java.util.Map; +import com.cloud.utils.db.GenericDaoBase; public class MockConfigurationDaoImpl extends GenericDaoBase implements ConfigurationDao { @@ -117,4 +119,8 @@ public class MockConfigurationDaoImpl extends GenericDaoBase searchPartialConfigurations() { + return List.of(); + } } diff --git a/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java b/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java index 3bf1fb97e4d..808511e6fdf 100644 --- a/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java +++ b/server/src/test/java/org/apache/cloudstack/backup/BackupManagerTest.java @@ -16,22 +16,45 @@ // under the License. package org.apache.cloudstack.backup; +import com.cloud.alert.AlertManager; +import com.cloud.configuration.Resource; +import com.cloud.dc.DataCenterVO; +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.domain.Domain; import com.cloud.event.ActionEventUtils; +import com.cloud.event.UsageEventUtils; import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.ResourceAllocationException; import com.cloud.storage.Volume; import com.cloud.storage.VolumeApiService; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.VolumeDao; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; +import com.cloud.user.AccountVO; +import com.cloud.user.DomainManager; +import com.cloud.user.ResourceLimitService; +import com.cloud.user.User; +import com.cloud.user.UserVO; +import com.cloud.utils.DateUtil; import com.cloud.utils.Pair; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.fsm.NoTransitionException; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineManager; +import com.cloud.vm.dao.VMInstanceDao; import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.command.admin.backup.UpdateBackupOfferingCmd; +import org.apache.cloudstack.api.command.user.backup.CreateBackupScheduleCmd; +import org.apache.cloudstack.backup.dao.BackupDao; import org.apache.cloudstack.backup.dao.BackupOfferingDao; +import org.apache.cloudstack.backup.dao.BackupScheduleDao; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.impl.ConfigDepotImpl; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -42,13 +65,21 @@ import org.mockito.MockedStatic; import org.mockito.Mockito; import org.mockito.MockitoAnnotations; import org.mockito.Spy; +import org.springframework.test.util.ReflectionTestUtils; +import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TimeZone; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) @@ -72,9 +103,38 @@ public class BackupManagerTest { @Mock VolumeDao volumeDao; + @Mock + VMInstanceDao vmInstanceDao; + + @Mock + AccountManager accountManager; + + @Mock + DomainManager domainManager; + + @Mock + ResourceLimitService resourceLimitMgr; + + @Mock + BackupScheduleDao backupScheduleDao; + + @Mock + BackupDao backupDao; + + @Mock + DataCenterDao dataCenterDao; + + @Mock + AlertManager alertManager; + + private AccountVO account; + private UserVO user; + private String[] hostPossibleValues = {"127.0.0.1", "hostname"}; private String[] datastoresPossibleValues = {"e9804933-8609-4de3-bccc-6278072a496c", "datastore-name"}; private AutoCloseable closeable; + private ConfigDepotImpl configDepotImpl; + private boolean updatedConfigKeyDepot = false; @Before public void setup() throws Exception { @@ -97,11 +157,19 @@ public class BackupManagerTest { offering.setUserDrivenBackupAllowed(true); return true; }); + + Account account = mock(Account.class); + User user = mock(User.class); + CallContext.register(user, account); } @After public void tearDown() throws Exception { closeable.close(); + if (updatedConfigKeyDepot) { + ReflectionTestUtils.setField(BackupManager.BackupFrameworkEnabled, "s_depot", configDepotImpl); + } + CallContext.unregister(); } @Test @@ -169,7 +237,7 @@ public class BackupManagerTest { Mockito.when(backupProvider.restoreBackedUpVolume(Mockito.any(), Mockito.eq(volumeUuid), Mockito.eq("127.0.0.1"), Mockito.eq("e9804933-8609-4de3-bccc-6278072a496c"), Mockito.eq(vmNameAndState))).thenReturn(new Pair(Boolean.TRUE, "Success")); - Pair restoreBackedUpVolume = backupManager.restoreBackedUpVolume(volumeUuid, backupVO, backupProvider, hostPossibleValues, datastoresPossibleValues, vm); + Pair restoreBackedUpVolume = backupManager.restoreBackedUpVolume(volumeUuid, backupVO, backupProvider, hostPossibleValues, datastoresPossibleValues, vm); assertEquals(Boolean.TRUE, restoreBackedUpVolume.first()); assertEquals("Success", restoreBackedUpVolume.second()); @@ -190,7 +258,7 @@ public class BackupManagerTest { Pair vmNameAndState = new Pair<>("i-2-3-VM", VirtualMachine.State.Running); Mockito.when(backupProvider.restoreBackedUpVolume(Mockito.any(), Mockito.eq(volumeUuid), Mockito.eq("127.0.0.1"), Mockito.eq("datastore-name"), Mockito.eq(vmNameAndState))).thenReturn(new Pair(Boolean.TRUE, "Success2")); - Pair restoreBackedUpVolume = backupManager.restoreBackedUpVolume(volumeUuid, backupVO, backupProvider, hostPossibleValues, datastoresPossibleValues, vm); + Pair restoreBackedUpVolume = backupManager.restoreBackedUpVolume(volumeUuid, backupVO, backupProvider, hostPossibleValues, datastoresPossibleValues, vm); assertEquals(Boolean.TRUE, restoreBackedUpVolume.first()); assertEquals("Success2", restoreBackedUpVolume.second()); @@ -211,8 +279,8 @@ public class BackupManagerTest { Pair vmNameAndState = new Pair<>("i-2-3-VM", VirtualMachine.State.Running); Mockito.when(backupProvider.restoreBackedUpVolume(Mockito.any(), Mockito.eq(volumeUuid), - Mockito.eq("hostname"), Mockito.eq("e9804933-8609-4de3-bccc-6278072a496c"), Mockito.eq(vmNameAndState) )).thenReturn(new Pair(Boolean.TRUE, "Success3")); - Pair restoreBackedUpVolume = backupManager.restoreBackedUpVolume(volumeUuid, backupVO, backupProvider, hostPossibleValues, datastoresPossibleValues, vm); + Mockito.eq("hostname"), Mockito.eq("e9804933-8609-4de3-bccc-6278072a496c"), Mockito.eq(vmNameAndState))).thenReturn(new Pair(Boolean.TRUE, "Success3")); + Pair restoreBackedUpVolume = backupManager.restoreBackedUpVolume(volumeUuid, backupVO, backupProvider, hostPossibleValues, datastoresPossibleValues, vm); assertEquals(Boolean.TRUE, restoreBackedUpVolume.first()); assertEquals("Success3", restoreBackedUpVolume.second()); @@ -233,8 +301,8 @@ public class BackupManagerTest { Pair vmNameAndState = new Pair<>("i-2-3-VM", VirtualMachine.State.Running); Mockito.when(backupProvider.restoreBackedUpVolume(Mockito.any(), Mockito.eq(volumeUuid), - Mockito.eq("hostname"), Mockito.eq("datastore-name"), Mockito.eq(vmNameAndState))).thenReturn(new Pair(Boolean.TRUE, "Success4")); - Pair restoreBackedUpVolume = backupManager.restoreBackedUpVolume(volumeUuid, backupVO, backupProvider, hostPossibleValues, datastoresPossibleValues, vm); + Mockito.eq("hostname"), Mockito.eq("datastore-name"), Mockito.eq(vmNameAndState))).thenReturn(new Pair(Boolean.TRUE, "Success4")); + Pair restoreBackedUpVolume = backupManager.restoreBackedUpVolume(volumeUuid, backupVO, backupProvider, hostPossibleValues, datastoresPossibleValues, vm); assertEquals(Boolean.TRUE, restoreBackedUpVolume.first()); assertEquals("Success4", restoreBackedUpVolume.second()); @@ -304,4 +372,289 @@ public class BackupManagerTest { } } } + + private void overrideBackupFrameworkConfigValue() { + ConfigKey configKey = BackupManager.BackupFrameworkEnabled; + this.configDepotImpl = (ConfigDepotImpl) ReflectionTestUtils.getField(configKey, "s_depot"); + ConfigDepotImpl configDepot = Mockito.mock(ConfigDepotImpl.class); + Mockito.when(configDepot.getConfigStringValue(Mockito.eq(BackupManager.BackupFrameworkEnabled.key()), + Mockito.eq(ConfigKey.Scope.Global), Mockito.isNull())).thenReturn("true"); + Mockito.when(configDepot.getConfigStringValue(Mockito.eq(BackupManager.BackupFrameworkEnabled.key()), + Mockito.eq(ConfigKey.Scope.Zone), Mockito.anyLong())).thenReturn("true"); + Mockito.when(configDepot.getConfigStringValue(Mockito.eq(BackupManager.BackupProviderPlugin.key()), + Mockito.eq(ConfigKey.Scope.Zone), Mockito.anyLong())).thenReturn("testbackupprovider"); + ReflectionTestUtils.setField(configKey, "s_depot", configDepot); + updatedConfigKeyDepot = true; + } + + @Test + public void testConfigureBackupSchedule() { + Long vmId = 1L; + Long zoneId = 2L; + Long accountId = 3L; + Long domainId = 4L; + Long backupOfferingId = 5L; + + CreateBackupScheduleCmd cmd = Mockito.mock(CreateBackupScheduleCmd.class); + when(cmd.getVmId()).thenReturn(vmId); + when(cmd.getTimezone()).thenReturn("GMT"); + when(cmd.getIntervalType()).thenReturn(DateUtil.IntervalType.DAILY); + when(cmd.getMaxBackups()).thenReturn(8); + when(cmd.getSchedule()).thenReturn("00:00:00"); + + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + when(vmInstanceDao.findById(vmId)).thenReturn(vm); + when(vm.getDataCenterId()).thenReturn(zoneId); + when(vm.getAccountId()).thenReturn(accountId); + when(vm.getBackupOfferingId()).thenReturn(backupOfferingId); + + overrideBackupFrameworkConfigValue(); + + Account account = Mockito.mock(Account.class); + when(accountManager.getAccount(accountId)).thenReturn(account); + when(account.getDomainId()).thenReturn(domainId); + Domain domain = Mockito.mock(Domain.class); + when(domainManager.getDomain(domainId)).thenReturn(domain); + when(resourceLimitMgr.findCorrectResourceLimitForAccount(account, Resource.ResourceType.backup, null)).thenReturn(8L); + when(resourceLimitMgr.findCorrectResourceLimitForDomain(domain, Resource.ResourceType.backup, null)).thenReturn(8L); + + BackupOfferingVO offering = Mockito.mock(BackupOfferingVO.class); + when(backupOfferingDao.findById(backupOfferingId)).thenReturn(offering); + when(offering.isUserDrivenBackupAllowed()).thenReturn(true); + when(offering.getProvider()).thenReturn("test"); + + BackupScheduleVO schedule = mock(BackupScheduleVO.class); + when(backupScheduleDao.findByVMAndIntervalType(vmId, DateUtil.IntervalType.DAILY)).thenReturn(schedule); + + backupManager.configureBackupSchedule(cmd); + + verify(schedule, times(1)).setScheduleType((short) DateUtil.IntervalType.DAILY.ordinal()); + verify(schedule, times(1)).setSchedule("00:00:00"); + verify(schedule, times(1)).setTimezone(TimeZone.getTimeZone("GMT").getID()); + verify(schedule, times(1)).setMaxBackups(8); + } + + @Test + public void testConfigureBackupScheduleLimitReached() { + Long vmId = 1L; + Long zoneId = 2L; + Long accountId = 3L; + Long domainId = 4L; + + CreateBackupScheduleCmd cmd = Mockito.mock(CreateBackupScheduleCmd.class); + when(cmd.getVmId()).thenReturn(vmId); + when(cmd.getTimezone()).thenReturn("GMT"); + when(cmd.getIntervalType()).thenReturn(DateUtil.IntervalType.DAILY); + when(cmd.getMaxBackups()).thenReturn(8); + + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + when(vmInstanceDao.findById(vmId)).thenReturn(vm); + when(vm.getDataCenterId()).thenReturn(zoneId); + when(vm.getAccountId()).thenReturn(accountId); + + overrideBackupFrameworkConfigValue(); + + Account account = Mockito.mock(Account.class); + when(accountManager.getAccount(accountId)).thenReturn(account); + when(account.getDomainId()).thenReturn(domainId); + Domain domain = Mockito.mock(Domain.class); + when(domainManager.getDomain(domainId)).thenReturn(domain); + when(resourceLimitMgr.findCorrectResourceLimitForAccount(account, Resource.ResourceType.backup, null)).thenReturn(10L); + when(resourceLimitMgr.findCorrectResourceLimitForDomain(domain, Resource.ResourceType.backup, null)).thenReturn(1L); + + InvalidParameterValueException exception = Assert.assertThrows(InvalidParameterValueException.class, + () -> backupManager.configureBackupSchedule(cmd)); + Assert.assertEquals(exception.getMessage(), "Max number of backups shouldn't exceed the domain/account level backup limit"); + } + + @Test + public void testCreateScheduledBackup() throws ResourceAllocationException { + Long vmId = 1L; + Long zoneId = 2L; + Long scheduleId = 3L; + Long backupOfferingId = 4L; + Long accountId = 5L; + Long backupId = 6L; + Long oldestBackupId = 7L; + Long newBackupSize = 1000000000L; + Long oldBackupSize = 400000000L; + + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + when(vmInstanceDao.findById(vmId)).thenReturn(vm); + when(vmInstanceDao.findByIdIncludingRemoved(vmId)).thenReturn(vm); + when(vm.getId()).thenReturn(vmId); + when(vm.getDataCenterId()).thenReturn(zoneId); + when(vm.getBackupOfferingId()).thenReturn(backupOfferingId); + when(vm.getAccountId()).thenReturn(accountId); + + overrideBackupFrameworkConfigValue(); + BackupOfferingVO offering = Mockito.mock(BackupOfferingVO.class); + when(backupOfferingDao.findById(backupOfferingId)).thenReturn(offering); + when(offering.isUserDrivenBackupAllowed()).thenReturn(true); + when(offering.getProvider()).thenReturn("test"); + + Account account = Mockito.mock(Account.class); + when(accountManager.getAccount(accountId)).thenReturn(account); + + BackupScheduleVO schedule = mock(BackupScheduleVO.class); + when(schedule.getScheduleType()).thenReturn(DateUtil.IntervalType.DAILY); + when(schedule.getMaxBackups()).thenReturn(0); + when(backupScheduleDao.findById(scheduleId)).thenReturn(schedule); + when(backupScheduleDao.findByVMAndIntervalType(vmId, DateUtil.IntervalType.DAILY)).thenReturn(schedule); + + BackupProvider backupProvider = mock(BackupProvider.class); + Backup backup = mock(Backup.class); + when(backup.getId()).thenReturn(backupId); + when(backup.getSize()).thenReturn(newBackupSize); + when(backupProvider.getName()).thenReturn("test"); + when(backupProvider.takeBackup(vm)).thenReturn(new Pair<>(true, backup)); + Map backupProvidersMap = new HashMap<>(); + backupProvidersMap.put(backupProvider.getName().toLowerCase(), backupProvider); + ReflectionTestUtils.setField(backupManager, "backupProvidersMap", backupProvidersMap); + + BackupVO backupVO = mock(BackupVO.class); + when(backupVO.getId()).thenReturn(backupId); + BackupVO oldestBackupVO = mock(BackupVO.class); + when(oldestBackupVO.getSize()).thenReturn(oldBackupSize); + when(oldestBackupVO.getId()).thenReturn(oldestBackupId); + when(oldestBackupVO.getVmId()).thenReturn(vmId); + when(oldestBackupVO.getBackupOfferingId()).thenReturn(backupOfferingId); + + when(backupDao.findById(backupId)).thenReturn(backupVO); + List backups = new ArrayList<>(List.of(oldestBackupVO)); + when(backupDao.listBackupsByVMandIntervalType(vmId, Backup.Type.DAILY)).thenReturn(backups); + when(backupDao.findByIdIncludingRemoved(oldestBackupId)).thenReturn(oldestBackupVO); + when(backupOfferingDao.findByIdIncludingRemoved(backupOfferingId)).thenReturn(offering); + when(backupProvider.deleteBackup(oldestBackupVO, false)).thenReturn(true); + when(backupDao.remove(oldestBackupVO.getId())).thenReturn(true); + + try (MockedStatic ignored = Mockito.mockStatic(ActionEventUtils.class)) { + Mockito.when(ActionEventUtils.onActionEvent(Mockito.anyLong(), Mockito.anyLong(), + Mockito.anyLong(), + Mockito.anyString(), Mockito.anyString(), + Mockito.anyLong(), Mockito.anyString())).thenReturn(1L); + + Assert.assertEquals(backupManager.createBackup(vmId, scheduleId), true); + + Mockito.verify(resourceLimitMgr, times(1)).incrementResourceCount(accountId, Resource.ResourceType.backup); + Mockito.verify(resourceLimitMgr, times(1)).incrementResourceCount(accountId, Resource.ResourceType.backup_storage, newBackupSize); + Mockito.verify(backupDao, times(1)).update(backupVO.getId(), backupVO); + + Mockito.verify(resourceLimitMgr, times(1)).decrementResourceCount(accountId, Resource.ResourceType.backup); + Mockito.verify(resourceLimitMgr, times(1)).decrementResourceCount(accountId, Resource.ResourceType.backup_storage, oldBackupSize); + Mockito.verify(backupDao, times(1)).remove(oldestBackupId); + } + } + + @Test (expected = ResourceAllocationException.class) + public void testCreateBackupLimitReached() throws ResourceAllocationException { + Long vmId = 1L; + Long zoneId = 2L; + Long scheduleId = 3L; + Long backupOfferingId = 4L; + Long accountId = 5L; + + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + when(vmInstanceDao.findById(vmId)).thenReturn(vm); + when(vm.getDataCenterId()).thenReturn(zoneId); + when(vm.getBackupOfferingId()).thenReturn(backupOfferingId); + when(vm.getAccountId()).thenReturn(accountId); + + overrideBackupFrameworkConfigValue(); + BackupOfferingVO offering = Mockito.mock(BackupOfferingVO.class); + when(backupOfferingDao.findById(backupOfferingId)).thenReturn(offering); + when(offering.isUserDrivenBackupAllowed()).thenReturn(true); + + BackupScheduleVO schedule = mock(BackupScheduleVO.class); + when(schedule.getScheduleType()).thenReturn(DateUtil.IntervalType.DAILY); + when(backupScheduleDao.findById(scheduleId)).thenReturn(schedule); + + Account account = Mockito.mock(Account.class); + when(account.getId()).thenReturn(accountId); + when(accountManager.getAccount(accountId)).thenReturn(account); + Mockito.doThrow(new ResourceAllocationException("", Resource.ResourceType.backup_storage)).when(resourceLimitMgr).checkResourceLimit(account, Resource.ResourceType.backup_storage, 0L); + + backupManager.createBackup(vmId, scheduleId); + + String msg = "Backup storage space resource limit exceeded for account id : " + accountId + ". Failed to create backup"; + Mockito.verify(alertManager, times(1)).sendAlert(AlertManager.AlertType.ALERT_TYPE_UPDATE_RESOURCE_COUNT, 0L, 0L, msg, "Backup storage space resource limit exceeded for account id : " + accountId + + ". Failed to create backups; please use updateResourceLimit to increase the limit"); + } + + @Test + public void testBackupSyncTask() { + Long dataCenterId = 1L; + Long vmId = 2L; + Long accountId = 3L; + Long backup2Id = 4L; + String restorePoint1ExternalId = "1234"; + Long backup1Size = 1 * Resource.ResourceType.bytesToGiB; + Long backup2Size = 2 * Resource.ResourceType.bytesToGiB; + Long newBackupSize = 3 * Resource.ResourceType.bytesToGiB; + Long metricSize = 4 * Resource.ResourceType.bytesToGiB; + + overrideBackupFrameworkConfigValue(); + + DataCenterVO dataCenter = mock(DataCenterVO.class); + when(dataCenter.getId()).thenReturn(dataCenterId); + when(dataCenterDao.listAllZones()).thenReturn(List.of(dataCenter)); + + BackupProvider backupProvider = mock(BackupProvider.class); + when(backupProvider.getName()).thenReturn("testbackupprovider"); + backupManager.setBackupProviders(List.of(backupProvider)); + backupManager.start(); + + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + when(vm.getId()).thenReturn(vmId); + when(vm.getAccountId()).thenReturn(accountId); + when(vmInstanceDao.listByZoneWithBackups(dataCenterId, null)).thenReturn(List.of(vm)); + Backup.Metric metric = new Backup.Metric(metricSize, null); + Map metricMap = new HashMap<>(); + metricMap.put(vm, metric); + when(backupProvider.getBackupMetrics(Mockito.anyLong(), Mockito.anyList())).thenReturn(metricMap); + + Backup.RestorePoint restorePoint1 = new Backup.RestorePoint(restorePoint1ExternalId, DateUtil.now(), "Root"); + Backup.RestorePoint restorePoint2 = new Backup.RestorePoint("12345", DateUtil.now(), "Root"); + List restorePoints = new ArrayList<>(List.of(restorePoint1, restorePoint2)); + when(backupProvider.listRestorePoints(vm)).thenReturn(restorePoints); + + BackupVO backupInDb1 = new BackupVO(); + backupInDb1.setSize(backup1Size); + backupInDb1.setExternalId(restorePoint1ExternalId); + + BackupVO backupInDb2 = new BackupVO(); + backupInDb2.setSize(backup2Size); + backupInDb2.setExternalId(null); + ReflectionTestUtils.setField(backupInDb2, "id", backup2Id); + when(backupDao.findById(backup2Id)).thenReturn(backupInDb2); + + when(backupDao.listByVmId(null, vmId)).thenReturn(List.of(backupInDb1, backupInDb2)); + + BackupVO newBackupEntry = new BackupVO(); + newBackupEntry.setSize(newBackupSize); + when(backupProvider.createNewBackupEntryForRestorePoint(restorePoint2, vm, metric)).thenReturn(newBackupEntry); + + try (MockedStatic ignored = Mockito.mockStatic(ActionEventUtils.class)) { + Mockito.when(ActionEventUtils.onActionEvent(Mockito.anyLong(), Mockito.anyLong(), + Mockito.anyLong(), + Mockito.anyString(), Mockito.anyString(), + Mockito.anyLong(), Mockito.anyString())).thenReturn(1L); + + try (MockedStatic ignored2 = Mockito.mockStatic(UsageEventUtils.class)) { + + BackupManagerImpl.BackupSyncTask backupSyncTask = backupManager.new BackupSyncTask(backupManager); + backupSyncTask.runInContext(); + + verify(resourceLimitMgr, times(1)).decrementResourceCount(accountId, Resource.ResourceType.backup_storage, backup1Size); + verify(resourceLimitMgr, times(1)).incrementResourceCount(accountId, Resource.ResourceType.backup_storage, metricSize); + Assert.assertEquals(backupInDb1.getSize(), metricSize); + + verify(resourceLimitMgr, times(1)).incrementResourceCount(accountId, Resource.ResourceType.backup); + verify(resourceLimitMgr, times(1)).incrementResourceCount(accountId, Resource.ResourceType.backup_storage, newBackupSize); + + verify(resourceLimitMgr, times(1)).decrementResourceCount(accountId, Resource.ResourceType.backup); + verify(resourceLimitMgr, times(1)).decrementResourceCount(accountId, Resource.ResourceType.backup_storage, backup2Size); + } + } + } } diff --git a/server/src/test/java/org/apache/cloudstack/storage/object/BucketApiServiceImplTest.java b/server/src/test/java/org/apache/cloudstack/storage/object/BucketApiServiceImplTest.java new file mode 100644 index 00000000000..3ce855b504b --- /dev/null +++ b/server/src/test/java/org/apache/cloudstack/storage/object/BucketApiServiceImplTest.java @@ -0,0 +1,182 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.object; + +import org.apache.cloudstack.api.command.user.bucket.CreateBucketCmd; +import org.apache.cloudstack.api.command.user.bucket.UpdateBucketCmd; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; + +import com.cloud.agent.api.to.BucketTO; +import com.cloud.configuration.Resource; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.resourcelimit.ResourceLimitManagerImpl; +import com.cloud.storage.BucketVO; +import com.cloud.storage.DataStoreRole; +import com.cloud.storage.dao.BucketDao; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; + +@RunWith(MockitoJUnitRunner.class) +public class BucketApiServiceImplTest { + @Spy + @InjectMocks + BucketApiServiceImpl bucketApiService; + + @Mock + AccountManager accountManager; + + @Mock + ObjectStoreDao objectStoreDao; + + @Mock + DataStoreManager dataStoreMgr; + + @Mock + private ResourceLimitManagerImpl resourceLimitManager; + + @Mock + private BucketDao bucketDao; + + @Test + public void testAllocBucket() throws ResourceAllocationException { + String bucketName = "bucket1"; + Long accountId = 1L; + Long poolId = 2L; + Long objectStoreId = 3L; + + CreateBucketCmd cmd = Mockito.mock(CreateBucketCmd.class); + Mockito.when(cmd.getBucketName()).thenReturn(bucketName); + Mockito.when(cmd.getEntityOwnerId()).thenReturn(accountId); + Mockito.when(cmd.getObjectStoragePoolId()).thenReturn(poolId); + Mockito.when(cmd.getQuota()).thenReturn(1); + + Account account = Mockito.mock(Account.class); + Mockito.when(accountManager.getActiveAccountById(accountId)).thenReturn(account); + + ObjectStoreVO objectStoreVO = Mockito.mock(ObjectStoreVO.class); + Mockito.when(objectStoreVO.getId()).thenReturn(objectStoreId); + Mockito.when(objectStoreDao.findById(poolId)).thenReturn(objectStoreVO); + ObjectStoreEntity objectStore = Mockito.mock(ObjectStoreEntity.class); + Mockito.when(dataStoreMgr.getDataStore(objectStoreId, DataStoreRole.Object)).thenReturn(objectStore); + Mockito.when(objectStore.createUser(accountId)).thenReturn(true); + + bucketApiService.allocBucket(cmd); + + Mockito.verify(resourceLimitManager, Mockito.times(1)).checkResourceLimit(account, Resource.ResourceType.bucket); + Mockito.verify(resourceLimitManager, Mockito.times(1)).checkResourceLimit(account, Resource.ResourceType.object_storage, 1 * Resource.ResourceType.bytesToGiB); + } + + @Test + public void testCreateBucket() { + Long objectStoreId = 1L; + Long poolId = 2L; + Long bucketId = 3L; + Long accountId = 4L; + String bucketName = "bucket1"; + + CreateBucketCmd cmd = Mockito.mock(CreateBucketCmd.class); + Mockito.when(cmd.getObjectStoragePoolId()).thenReturn(poolId); + Mockito.when(cmd.getEntityId()).thenReturn(bucketId); + Mockito.when(cmd.getQuota()).thenReturn(1); + + BucketVO bucket = new BucketVO(bucketName); + Mockito.when(bucketDao.findById(bucketId)).thenReturn(bucket); + ReflectionTestUtils.setField(bucket, "accountId", accountId); + + ObjectStoreVO objectStoreVO = Mockito.mock(ObjectStoreVO.class); + Mockito.when(objectStoreVO.getId()).thenReturn(objectStoreId); + Mockito.when(objectStoreDao.findById(poolId)).thenReturn(objectStoreVO); + ObjectStoreEntity objectStore = Mockito.mock(ObjectStoreEntity.class); + Mockito.when(dataStoreMgr.getDataStore(objectStoreId, DataStoreRole.Object)).thenReturn(objectStore); + Mockito.when(objectStore.createBucket(bucket, false)).thenReturn(bucket); + + bucketApiService.createBucket(cmd); + + Mockito.verify(resourceLimitManager, Mockito.times(1)).incrementResourceCount(accountId, Resource.ResourceType.bucket); + Mockito.verify(resourceLimitManager, Mockito.times(1)).incrementResourceCount(accountId, Resource.ResourceType.object_storage, 1 * Resource.ResourceType.bytesToGiB); + Assert.assertEquals(bucket.getState(), Bucket.State.Created); + } + + @Test + public void testDeleteBucket() { + Long bucketId = 1L; + Long accountId = 2L; + Long objectStoreId = 3L; + String bucketName = "bucket1"; + + BucketVO bucket = new BucketVO(bucketName); + Mockito.when(bucketDao.findById(bucketId)).thenReturn(bucket); + ReflectionTestUtils.setField(bucket, "objectStoreId", objectStoreId); + ReflectionTestUtils.setField(bucket, "quota", 1); + ReflectionTestUtils.setField(bucket, "accountId", accountId); + + ObjectStoreVO objectStoreVO = Mockito.mock(ObjectStoreVO.class); + Mockito.when(objectStoreVO.getId()).thenReturn(objectStoreId); + Mockito.when(objectStoreDao.findById(objectStoreId)).thenReturn(objectStoreVO); + ObjectStoreEntity objectStore = Mockito.mock(ObjectStoreEntity.class); + Mockito.when(dataStoreMgr.getDataStore(objectStoreId, DataStoreRole.Object)).thenReturn(objectStore); + Mockito.when(objectStore.deleteBucket(Mockito.any(BucketTO.class))).thenReturn(true); + + bucketApiService.deleteBucket(bucketId, null); + + Mockito.verify(resourceLimitManager, Mockito.times(1)).decrementResourceCount(accountId, Resource.ResourceType.bucket); + Mockito.verify(resourceLimitManager, Mockito.times(1)).decrementResourceCount(accountId, Resource.ResourceType.object_storage, 1 * Resource.ResourceType.bytesToGiB); + } + + @Test + public void testUpdateBucket() throws ResourceAllocationException { + Long bucketId = 1L; + Long objectStoreId = 2L; + Long accountId = 3L; + Integer bucketQuota = 2; + Integer cmdQuota = 1; + String bucketName = "bucket1"; + + UpdateBucketCmd cmd = Mockito.mock(UpdateBucketCmd.class); + Mockito.when(cmd.getId()).thenReturn(bucketId); + Mockito.when(cmd.getQuota()).thenReturn(cmdQuota); + + BucketVO bucket = new BucketVO(bucketName); + ReflectionTestUtils.setField(bucket, "quota", bucketQuota); + ReflectionTestUtils.setField(bucket, "accountId", accountId); + ReflectionTestUtils.setField(bucket, "objectStoreId", objectStoreId); + Mockito.when(bucketDao.findById(bucketId)).thenReturn(bucket); + + Account account = Mockito.mock(Account.class); + + ObjectStoreVO objectStoreVO = Mockito.mock(ObjectStoreVO.class); + Mockito.when(objectStoreVO.getId()).thenReturn(objectStoreId); + Mockito.when(objectStoreDao.findById(objectStoreId)).thenReturn(objectStoreVO); + ObjectStoreEntity objectStore = Mockito.mock(ObjectStoreEntity.class); + Mockito.when(dataStoreMgr.getDataStore(objectStoreId, DataStoreRole.Object)).thenReturn(objectStore); + + bucketApiService.updateBucket(cmd, null); + + Mockito.verify(resourceLimitManager, Mockito.times(1)).decrementResourceCount(accountId, Resource.ResourceType.object_storage, (bucketQuota - cmdQuota) * Resource.ResourceType.bytesToGiB); + } +} diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResource.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResource.java index ab55f654515..8c7d082c33e 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResource.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResource.java @@ -40,11 +40,6 @@ public class LocalNfsSecondaryStorageResource extends NfsSecondaryStorageResourc this._inSystemVM = false; } - @Override - public void setParentPath(String path) { - this._parent = path; - } - @Override public Answer executeRequest(Command cmd) { return super.executeRequest(cmd); @@ -57,7 +52,7 @@ public class LocalNfsSecondaryStorageResource extends NfsSecondaryStorageResourc String dir = mountUri(uri, nfsVersion); return _parent + "/" + dir; } catch (Exception e) { - String msg = "GetRootDir for " + secUrl + " failed due to " + e.toString(); + String msg = "GetRootDir for " + secUrl + " failed due to " + e; logger.error(msg, e); throw new CloudRuntimeException(msg); } @@ -75,14 +70,14 @@ public class LocalNfsSecondaryStorageResource extends NfsSecondaryStorageResourc // Change permissions for the mountpoint - seems to bypass authentication Script script = new Script(true, "chmod", _timeout, logger); - script.add("777", localRootPath); + script.add("1777", localRootPath); String result = script.execute(); if (result != null) { String errMsg = "Unable to set permissions for " + localRootPath + " due to " + result; logger.error(errMsg); throw new CloudRuntimeException(errMsg); } - logger.debug("Successfully set 777 permission for " + localRootPath); + logger.debug("Successfully set 1777 permission for " + localRootPath); // XXX: Adding the check for creation of snapshots dir here. Might have // to move it somewhere more logical later. diff --git a/systemvm/debian/opt/cloud/bin/cs/CsAddress.py b/systemvm/debian/opt/cloud/bin/cs/CsAddress.py index 0d1c31ac14b..1c4695df806 100755 --- a/systemvm/debian/opt/cloud/bin/cs/CsAddress.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsAddress.py @@ -19,15 +19,16 @@ import logging from netaddr import IPAddress, IPNetwork import subprocess import time + from . import CsHelper from .CsDatabag import CsDataBag from .CsApp import CsApache, CsDnsmasq, CsPasswdSvc from .CsRoute import CsRoute from .CsRule import CsRule +from .CsStaticRoutes import CsStaticRoutes VRRP_TYPES = ['guest'] - class CsAddress(CsDataBag): def compare(self): @@ -556,8 +557,10 @@ class CsIP: (self.dev, guestNetworkCidr, self.address['gateway'], self.dev)]) if self.is_private_gateway(): - self.fw.append(["filter", "", "-A FORWARD -d %s -o %s -j ACL_INBOUND_%s" % + self.fw.append(["filter", "front", "-A FORWARD -d %s -o %s -j ACL_INBOUND_%s" % (self.address['network'], self.dev, self.dev)]) + self.fw.append(["filter", "front", "-A FORWARD -d %s -o %s -m state --state RELATED,ESTABLISHED -j ACCEPT" % + (self.address['network'], self.dev)]) self.fw.append(["filter", "", "-A ACL_INBOUND_%s -j DROP" % self.dev]) self.fw.append(["mangle", "", "-A PREROUTING -m state --state NEW -i %s -s %s ! -d %s/32 -j ACL_OUTBOUND_%s" % @@ -565,6 +568,23 @@ class CsIP: self.fw.append(["mangle", "front", "-A PREROUTING -s %s -d %s -m state --state NEW -j MARK --set-xmark %s/0xffffffff" % (self.cl.get_vpccidr(), self.address['network'], hex(100 + int(self.dev[3:])))]) + + static_routes = CsStaticRoutes("staticroutes", self.config) + if static_routes: + for item in static_routes.get_bag(): + if item == "id": + continue + static_route = static_routes.get_bag()[item] + if static_route['ip_address'] == self.address['public_ip'] and not static_route['revoke']: + self.fw.append(["mangle", "", + "-A PREROUTING -m state --state NEW -i %s -s %s ! -d %s/32 -j ACL_OUTBOUND_%s" % + (self.dev, static_route['network'], static_route['ip_address'], self.dev)]) + self.fw.append(["filter", "front", "-A FORWARD -d %s -o %s -j ACL_INBOUND_%s" % + (static_route['network'], self.dev, self.dev)]) + self.fw.append(["filter", "front", + "-A FORWARD -d %s -o %s -m state --state RELATED,ESTABLISHED -j ACCEPT" % + (static_route['network'], self.dev)]) + if self.address["source_nat"]: self.fw.append(["nat", "front", "-A POSTROUTING -o %s -j SNAT --to-source %s" % diff --git a/systemvm/debian/opt/cloud/bin/cs/CsDhcp.py b/systemvm/debian/opt/cloud/bin/cs/CsDhcp.py index ce9493c5e69..a9b8253d1c1 100755 --- a/systemvm/debian/opt/cloud/bin/cs/CsDhcp.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsDhcp.py @@ -144,8 +144,7 @@ class CsDhcp(CsDataBag): # Listen Address if self.cl.is_redundant(): listen_address.append(gateway) - else: - listen_address.append(ip) + listen_address.append(ip) # Add localized "data-server" records in /etc/hosts for VPC routers if (self.config.is_vpc() or self.config.is_router()) and ('is_vr_guest_gateway' not in gn.data or (not gn.data['is_vr_guest_gateway'])): self.add_host(gateway, "%s data-server" % CsHelper.get_hostname()) diff --git a/test/integration/smoke/test_dynamicroles.py b/test/integration/smoke/test_dynamicroles.py index e404835fbb8..19248d28391 100644 --- a/test/integration/smoke/test_dynamicroles.py +++ b/test/integration/smoke/test_dynamicroles.py @@ -74,6 +74,7 @@ class TestData(object): "listApis": "allow", "listAccounts": "allow", "listClusters": "deny", + "*VmwareDc*": "allow", "*VM*": "allow", "*Host*": "deny" } diff --git a/ui/public/locales/de_DE.json b/ui/public/locales/de_DE.json index d9adc43469e..abf471ae17d 100644 --- a/ui/public/locales/de_DE.json +++ b/ui/public/locales/de_DE.json @@ -1265,7 +1265,6 @@ "label.save.new.rule": "Neue Regel Speichern", "label.schedule": "Zeitplan", "label.scheduled.backups": "geplante Backups", -"label.scheduled.snapshots": "geplante Schnappschüsse", "label.scope": "Geltungsbereich", "label.search": "Suche", "label.secondary.isolated.vlan.type.isolated": "Isoliert", diff --git a/ui/public/locales/el_GR.json b/ui/public/locales/el_GR.json index 754cde89542..7b7f4b29e12 100644 --- a/ui/public/locales/el_GR.json +++ b/ui/public/locales/el_GR.json @@ -1517,7 +1517,6 @@ "label.scale.vm": "Κλίμακα εικονικής μηχανής", "label.schedule": "Πρόγραμμα", "label.scheduled.backups": "Προγραμματισμένα αντίγραφα ασφαλείας", -"label.scheduled.snapshots": "Προγραμματισμένα στιγμιότυπα", "label.scope": "Πεδίο εφαρμογής", "label.search": "Αναζήτηση", "label.secondary.isolated.vlan.type.isolated": "Απομονωμένες", diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index 4a98e05219b..b449dcb2ead 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -56,6 +56,7 @@ "label.action.remove.nodes.from.kubernetes.cluster": "Remove nodes from Kubernetes cluster", "label.action.attach.disk": "Attach disk", "label.action.attach.iso": "Attach ISO", +"label.action.attach.to.instance": "Attach to Instance", "label.action.bulk.delete.egress.firewall.rules": "Bulk delete egress firewall rules", "label.action.bulk.delete.firewall.rules": "Bulk delete firewall rules", "label.action.bulk.delete.ip.v6.firewall.rules": "Bulk remove IPv6 firewall rules", @@ -403,9 +404,11 @@ "label.associatednetworkid": "Associated Network ID", "label.associatednetworkname": "Network name", "label.asyncbackup": "Async backup", +"label.attach.vol.to.instance": "Attach the created Volume to an existing Instance", "label.attaching": "Attaching", "label.authentication.method": "Authentication Method", "label.authentication.sshkey": "System SSH Key", +"label.use.existing.vcenter.credentials.from.zone": "Use existing vCenter credentials from the Zone", "label.autoscale": "AutoScale", "label.autoscalevmgroupname": "AutoScaling Group", "label.author.email": "Author e-mail", @@ -435,6 +438,9 @@ "label.backupofferingname": "Backup offering", "label.backup.repository.add": "Add backup repository", "label.backup.repository.remove": "Remove backup repository", +"label.backuplimit": "Backup Limits", +"label.backup.storage": "Backup Storage", +"label.backupstoragelimit": "Backup Storage Limits (GiB)", "label.balance": "Balance", "label.bandwidth": "Bandwidth", "label.baremetal.dhcp.devices": "Bare metal DHCP devices", @@ -463,6 +469,7 @@ "label.brocade.vcs.address": "Vcs switch address", "label.browser": "Browser", "label.bucket": "Bucket", +"label.bucketlimit": "Bucket Limits", "label.by.account": "By Account", "label.by.domain": "By domain", "label.by.level": "By level", @@ -1087,7 +1094,7 @@ "label.hastate": "HA state", "label.headers": "Headers", "label.header.backup.schedule": "You can set up recurring backup schedules by selecting from the available options below and applying your policy preference.", -"label.header.volume.snapshot": "You can set up recurring Snapshot schedules by selecting from the available options below and applying your policy preference.", +"label.header.volume.snapshot": "You can set up recurring Snapshots by selecting from the available options below and applying your policy preference.", "label.header.volume.take.snapshot": "Please confirm that you want to take a Snapshot of this volume.", "label.health.check": "Health check", "label.heapmemoryused": "Heap-memory used", @@ -1408,6 +1415,10 @@ "label.max.primary.storage": "Max. primary (GiB)", "label.max.secondary.storage": "Max. secondary (GiB)", "label.max.migrations": "Max. migrations", +"label.maxbackup": "Max. Backups", +"label.maxbackupstorage": "Max. Backup Storage (GiB)", +"label.maxbackups.to.retain": "Max. Backups to retain", +"label.maxbucket": "Max. Buckets", "label.maxcpu": "Max. CPU cores", "label.maxcpunumber": "Max CPU cores", "label.maxdatavolumeslimit": "Max data volumes limit", @@ -1420,6 +1431,7 @@ "label.maxmembers": "Max members", "label.maxmemory": "Max. memory (MiB)", "label.maxnetwork": "Max. Networks", +"label.maxobjectstorage": "Max. Object Storage (GiB)", "label.maxprimarystorage": "Max. primary storage (GiB)", "label.maxproject": "Max. projects", "label.maxpublicip": "Max. public IPs", @@ -1608,6 +1620,7 @@ "label.oauth.verification": "OAuth verification", "label.ocfs2": "OCFS2", "label.object.storage" : "Object Storage", +"label.objectstoragelimit": "Object Storage Limits (GiB)", "label.object.presigned.url": "Presigned URL", "label.object.presigned.url.description" : "Presigned URL of the object in order to access it without authentication.", "label.object.url.description" : "URL of the object", @@ -2028,7 +2041,6 @@ "label.schedule": "Schedule", "label.schedule.add": "Add schedule", "label.scheduled.backups": "Scheduled backups", -"label.scheduled.snapshots": "Scheduled Snapshots", "label.schedules": "Schedules", "label.scope": "Scope", "label.scope.tooltip": "Primary Storage Pool Scope", @@ -2466,7 +2478,7 @@ "label.users": "Users", "label.usersource": "User type", "label.using.cli": "Using CLI", -"label.utilization": "Utilisation", +"label.utilization": "Utilization", "label.uuid": "ID", "label.value": "Value", "label.vcenter": "VMware datacenter vCenter", @@ -2650,7 +2662,7 @@ "label.objectstorageid": "Object Storage Pool", "label.bucket.update": "Update Bucket", "label.bucket.delete": "Delete Bucket", -"label.quotagb": "Quota in GB", +"label.quotagib": "Quota in GiB", "label.encryption": "Encryption", "label.etcdnodes": "Number of etcd nodes", "label.versioning": "Versioning", @@ -3306,6 +3318,7 @@ "message.license.agreements.not.accepted": "License agreements not accepted.", "message.linstor.resourcegroup.description": "Linstor resource group to use for primary storage.", "message.list.zone.vmware.datacenter.empty": "No VMware Datacenter exists in the selected Zone", +"message.list.zone.vmware.hosts.empty": "No VMware hosts were found in the selected Datacenter", "message.listnsp.not.return.providerid": "error: listNetworkServiceProviders API doesn't return VirtualRouter provider ID.", "message.load.host.failed": "Failed to load hosts.", "message.loadbalancer.stickypolicy.configuration": "Customize the load balancer stickiness policy:", diff --git a/ui/public/locales/ja_JP.json b/ui/public/locales/ja_JP.json index f1518286f80..53009781500 100644 --- a/ui/public/locales/ja_JP.json +++ b/ui/public/locales/ja_JP.json @@ -1969,7 +1969,6 @@ "label.scaleup.policy": "スケールアップポリシー", "label.schedule": "スケジュール", "label.scheduled.backups": "スケジュールされたバックアップ", - "label.scheduled.snapshots": "スケジュールされたスナップショット", "label.scope": "スコープ", "label.search": "検索", "label.secondary.isolated.vlan.type.isolated": "隔離", diff --git a/ui/public/locales/ko_KR.json b/ui/public/locales/ko_KR.json index a34cdea767f..153bfd64cef 100644 --- a/ui/public/locales/ko_KR.json +++ b/ui/public/locales/ko_KR.json @@ -1322,7 +1322,6 @@ "label.scale.vm": "VM \ud655\uc7a5", "label.schedule": "\uc2a4\ucf00\uc904", "label.scheduled.backups": "\uc608\uc57d\ub41c \ubc31\uc5c5", -"label.scheduled.snapshots": "\uc608\uc57d\ub41c \uc2a4\ub0c5\uc0f7", "label.scope": "\ubc94\uc704", "label.search": "\uac80\uc0c9", "label.secondary.isolated.vlan.type.isolated": "isolated", diff --git a/ui/public/locales/pt_BR.json b/ui/public/locales/pt_BR.json index 511c8714daa..9de19db7442 100644 --- a/ui/public/locales/pt_BR.json +++ b/ui/public/locales/pt_BR.json @@ -1430,7 +1430,6 @@ "label.scale.vm": "Escalar VM", "label.schedule": "Programar", "label.scheduled.backups": "Backups programados", -"label.scheduled.snapshots": "Snapshots programados", "label.scope": "Escopo", "label.search": "Pesquisar", "label.secondary.isolated.vlan.type.isolated": "Isolada", diff --git a/ui/public/locales/zh_CN.json b/ui/public/locales/zh_CN.json index 91353d2f81a..5337829b53d 100644 --- a/ui/public/locales/zh_CN.json +++ b/ui/public/locales/zh_CN.json @@ -2248,7 +2248,6 @@ "label.schedule": "\u65E5\u7A0B", "label.scheduled.backups": "\u5B9A\u65F6\u5907\u4EFD", - "label.scheduled.snapshots": "\u8BA1\u5212\u5FEB\u7167", "label.scope": "\u8303\u56F4", "label.search": "\u641C\u7D22", diff --git a/ui/src/components/view/ListResourceTable.vue b/ui/src/components/view/ListResourceTable.vue index a95927f00cf..f8fdc5b5ada 100644 --- a/ui/src/components/view/ListResourceTable.vue +++ b/ui/src/components/view/ListResourceTable.vue @@ -51,7 +51,7 @@