diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index e90c75979b6..842e4497a4a 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -41,3 +41,9 @@ jobs: days-before-pr-close: 240 exempt-issue-labels: 'gsoc,good-first-issue,long-term-plan' exempt-pr-labels: 'status:ready-for-merge,status:needs-testing,status:on-hold' + - uses: actions/stale@v10 + with: + stale-issue-label: 'archive' + days-before-stale: 240 + exempt-issue-labels: 'gsoc,good-first-issue,long-term-plan' + days-before-close: -1 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 26adafcbf26..49829caf125 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -52,6 +52,16 @@ repos: args: ['644'] files: \.md$ stages: [manual] + - id: insert-license + name: add license for all cfg files + description: automatically adds a licence header to all cfg files that don't have a license header + files: \.cfg$ + args: + - --comment-style + - '|#|' + - --license-filepath + - .github/workflows/license-templates/LICENSE.txt + - --fuzzy-match-generates-todo - id: insert-license name: add license for all Markdown files files: \.md$ diff --git a/agent/src/main/java/com/cloud/agent/resource/consoleproxy/ConsoleProxyResource.java b/agent/src/main/java/com/cloud/agent/resource/consoleproxy/ConsoleProxyResource.java index cd9feaf93b4..ef98fa532ec 100644 --- a/agent/src/main/java/com/cloud/agent/resource/consoleproxy/ConsoleProxyResource.java +++ b/agent/src/main/java/com/cloud/agent/resource/consoleproxy/ConsoleProxyResource.java @@ -331,7 +331,7 @@ public class ConsoleProxyResource extends ServerResourceBase implements ServerRe final Object resource = this; logger.info("Building class loader for com.cloud.consoleproxy.ConsoleProxy"); if (consoleProxyMain == null) { - logger.info("Running com.cloud.consoleproxy.ConsoleProxy with encryptor password={}", encryptorPassword); + logger.info("Running com.cloud.consoleproxy.ConsoleProxy"); consoleProxyMain = new Thread(new ManagedContextRunnable() { @Override protected void runInContext() { diff --git a/api/src/main/java/com/cloud/event/EventTypes.java b/api/src/main/java/com/cloud/event/EventTypes.java index f1455d14f68..389fcc1da5c 100644 --- a/api/src/main/java/com/cloud/event/EventTypes.java +++ b/api/src/main/java/com/cloud/event/EventTypes.java @@ -584,6 +584,7 @@ public class EventTypes { // Network ACL public static final String EVENT_NETWORK_ACL_CREATE = "NETWORK.ACL.CREATE"; + public static final String EVENT_NETWORK_ACL_IMPORT = "NETWORK.ACL.IMPORT"; public static final String EVENT_NETWORK_ACL_DELETE = "NETWORK.ACL.DELETE"; public static final String EVENT_NETWORK_ACL_REPLACE = "NETWORK.ACL.REPLACE"; public static final String EVENT_NETWORK_ACL_UPDATE = "NETWORK.ACL.UPDATE"; diff --git a/api/src/main/java/com/cloud/network/NetworkService.java b/api/src/main/java/com/cloud/network/NetworkService.java index 585ebc89d42..742206c7e3b 100644 --- a/api/src/main/java/com/cloud/network/NetworkService.java +++ b/api/src/main/java/com/cloud/network/NetworkService.java @@ -108,6 +108,10 @@ public interface NetworkService { PhysicalNetwork physicalNetwork, long zoneId, ControlledEntity.ACLType aclType) throws InsufficientCapacityException, ConcurrentOperationException, ResourceAllocationException; + Network createGuestNetwork(long networkOfferingId, String name, String displayText, Account owner, + PhysicalNetwork physicalNetwork, long zoneId, ControlledEntity.ACLType aclType, Pair vrIfaceMTUs) throws + InsufficientCapacityException, ConcurrentOperationException, ResourceAllocationException; + Pair, Integer> searchForNetworks(ListNetworksCmd cmd); boolean deleteNetwork(long networkId, boolean forced); diff --git a/api/src/main/java/com/cloud/network/vpc/NetworkACLService.java b/api/src/main/java/com/cloud/network/vpc/NetworkACLService.java index 40aee1f08f1..84e48d5d5b8 100644 --- a/api/src/main/java/com/cloud/network/vpc/NetworkACLService.java +++ b/api/src/main/java/com/cloud/network/vpc/NetworkACLService.java @@ -19,6 +19,7 @@ package com.cloud.network.vpc; import java.util.List; import org.apache.cloudstack.api.command.user.network.CreateNetworkACLCmd; +import org.apache.cloudstack.api.command.user.network.ImportNetworkACLCmd; import org.apache.cloudstack.api.command.user.network.ListNetworkACLListsCmd; import org.apache.cloudstack.api.command.user.network.ListNetworkACLsCmd; import org.apache.cloudstack.api.command.user.network.MoveNetworkAclItemCmd; @@ -98,4 +99,6 @@ public interface NetworkACLService { NetworkACLItem moveNetworkAclRuleToNewPosition(MoveNetworkAclItemCmd moveNetworkAclItemCmd); NetworkACLItem moveRuleToTheTopInACLList(NetworkACLItem ruleBeingMoved); + + List importNetworkACLRules(ImportNetworkACLCmd cmd) throws ResourceUnavailableException; } diff --git a/api/src/main/java/com/cloud/server/ResourceTag.java b/api/src/main/java/com/cloud/server/ResourceTag.java index b3026deceff..32305753f1a 100644 --- a/api/src/main/java/com/cloud/server/ResourceTag.java +++ b/api/src/main/java/com/cloud/server/ResourceTag.java @@ -16,14 +16,14 @@ // under the License. package com.cloud.server; -import org.apache.cloudstack.acl.ControlledEntity; -import org.apache.cloudstack.api.Identity; -import org.apache.cloudstack.api.InternalIdentity; - import java.util.HashMap; import java.util.Locale; import java.util.Map; +import org.apache.cloudstack.acl.ControlledEntity; +import org.apache.cloudstack.api.Identity; +import org.apache.cloudstack.api.InternalIdentity; + public interface ResourceTag extends ControlledEntity, Identity, InternalIdentity { // FIXME - extract enum to another interface as its used both by resourceTags and resourceMetaData code @@ -70,7 +70,7 @@ public interface ResourceTag extends ControlledEntity, Identity, InternalIdentit GuestOs(false, true), NetworkOffering(false, true), VpcOffering(true, false), - Domain(false, false, true), + Domain(true, false, true), ObjectStore(false, false, true); diff --git a/api/src/main/java/com/cloud/storage/VolumeApiService.java b/api/src/main/java/com/cloud/storage/VolumeApiService.java index 19c2ebe455a..1a9bcc6ee98 100644 --- a/api/src/main/java/com/cloud/storage/VolumeApiService.java +++ b/api/src/main/java/com/cloud/storage/VolumeApiService.java @@ -56,9 +56,9 @@ public interface VolumeApiService { Boolean.class, "use.https.to.upload", "true", - "Determines the protocol (HTTPS or HTTP) ACS will use to generate links to upload ISOs, volumes, and templates. When set as 'true', ACS will use protocol HTTPS, otherwise, it will use protocol HTTP. Default value is 'true'.", + "Controls whether upload links for ISOs, volumes, and templates use HTTPS (true, default) or HTTP (false). After changing this setting, the Secondary Storage VM (SSVM) must be recreated", true, - ConfigKey.Scope.StoragePool); + ConfigKey.Scope.Zone); /** * Creates the database object for a volume based on the given criteria diff --git a/api/src/main/java/com/cloud/user/AccountService.java b/api/src/main/java/com/cloud/user/AccountService.java index 8f29a2fbc42..b92654bfe17 100644 --- a/api/src/main/java/com/cloud/user/AccountService.java +++ b/api/src/main/java/com/cloud/user/AccountService.java @@ -124,7 +124,7 @@ public interface AccountService { void validateAccountHasAccessToResource(Account account, AccessType accessType, Object resource); - Long finalyzeAccountId(String accountName, Long domainId, Long projectId, boolean enabledOnly); + Long finalizeAccountId(String accountName, Long domainId, Long projectId, boolean enabledOnly); /** * returns the user account object for a given user id diff --git a/api/src/main/java/org/apache/cloudstack/alert/AlertService.java b/api/src/main/java/org/apache/cloudstack/alert/AlertService.java index cc3188feeca..fcc87908bd5 100644 --- a/api/src/main/java/org/apache/cloudstack/alert/AlertService.java +++ b/api/src/main/java/org/apache/cloudstack/alert/AlertService.java @@ -24,18 +24,24 @@ import com.cloud.exception.InvalidParameterValueException; public interface AlertService { public static class AlertType { - private static Set defaultAlertTypes = new HashSet(); + private static final Set defaultAlertTypes = new HashSet<>(); private final String name; private final short type; + private final boolean repetitionAllowed; - private AlertType(short type, String name, boolean isDefault) { + private AlertType(short type, String name, boolean isDefault, boolean repetitionAllowed) { this.name = name; this.type = type; + this.repetitionAllowed = repetitionAllowed; if (isDefault) { defaultAlertTypes.add(this); } } + private AlertType(short type, String name, boolean isDefault) { + this(type, name, isDefault, false); + } + public static final AlertType ALERT_TYPE_MEMORY = new AlertType(Capacity.CAPACITY_TYPE_MEMORY, "ALERT.MEMORY", true); public static final AlertType ALERT_TYPE_CPU = new AlertType(Capacity.CAPACITY_TYPE_CPU, "ALERT.CPU", true); public static final AlertType ALERT_TYPE_STORAGE = new AlertType(Capacity.CAPACITY_TYPE_STORAGE, "ALERT.STORAGE", true); @@ -45,36 +51,36 @@ public interface AlertService { public static final AlertType ALERT_TYPE_VIRTUAL_NETWORK_IPV6_SUBNET = new AlertType(Capacity.CAPACITY_TYPE_VIRTUAL_NETWORK_IPV6_SUBNET, "ALERT.NETWORK.IPV6SUBNET", true); public static final AlertType ALERT_TYPE_PRIVATE_IP = new AlertType(Capacity.CAPACITY_TYPE_PRIVATE_IP, "ALERT.NETWORK.PRIVATEIP", true); public static final AlertType ALERT_TYPE_SECONDARY_STORAGE = new AlertType(Capacity.CAPACITY_TYPE_SECONDARY_STORAGE, "ALERT.STORAGE.SECONDARY", true); - public static final AlertType ALERT_TYPE_HOST = new AlertType((short)7, "ALERT.COMPUTE.HOST", true); - public static final AlertType ALERT_TYPE_USERVM = new AlertType((short)8, "ALERT.USERVM", true); - public static final AlertType ALERT_TYPE_DOMAIN_ROUTER = new AlertType((short)9, "ALERT.SERVICE.DOMAINROUTER", true); - public static final AlertType ALERT_TYPE_CONSOLE_PROXY = new AlertType((short)10, "ALERT.SERVICE.CONSOLEPROXY", true); + public static final AlertType ALERT_TYPE_HOST = new AlertType((short)7, "ALERT.COMPUTE.HOST", true, true); + public static final AlertType ALERT_TYPE_USERVM = new AlertType((short)8, "ALERT.USERVM", true, true); + public static final AlertType ALERT_TYPE_DOMAIN_ROUTER = new AlertType((short)9, "ALERT.SERVICE.DOMAINROUTER", true, true); + public static final AlertType ALERT_TYPE_CONSOLE_PROXY = new AlertType((short)10, "ALERT.SERVICE.CONSOLEPROXY", true, true); public static final AlertType ALERT_TYPE_ROUTING = new AlertType((short)11, "ALERT.NETWORK.ROUTING", true); - public static final AlertType ALERT_TYPE_STORAGE_MISC = new AlertType((short)12, "ALERT.STORAGE.MISC", true); + public static final AlertType ALERT_TYPE_STORAGE_MISC = new AlertType((short)12, "ALERT.STORAGE.MISC", true, true); public static final AlertType ALERT_TYPE_USAGE_SERVER = new AlertType((short)13, "ALERT.USAGE", true); - public static final AlertType ALERT_TYPE_MANAGEMENT_NODE = new AlertType((short)14, "ALERT.MANAGEMENT", true); + public static final AlertType ALERT_TYPE_MANAGEMENT_NODE = new AlertType((short)14, "ALERT.MANAGEMENT", true, true); public static final AlertType ALERT_TYPE_DOMAIN_ROUTER_MIGRATE = new AlertType((short)15, "ALERT.NETWORK.DOMAINROUTERMIGRATE", true); public static final AlertType ALERT_TYPE_CONSOLE_PROXY_MIGRATE = new AlertType((short)16, "ALERT.SERVICE.CONSOLEPROXYMIGRATE", true); public static final AlertType ALERT_TYPE_USERVM_MIGRATE = new AlertType((short)17, "ALERT.USERVM.MIGRATE", true); public static final AlertType ALERT_TYPE_VLAN = new AlertType((short)18, "ALERT.NETWORK.VLAN", true); - public static final AlertType ALERT_TYPE_SSVM = new AlertType((short)19, "ALERT.SERVICE.SSVM", true); + public static final AlertType ALERT_TYPE_SSVM = new AlertType((short)19, "ALERT.SERVICE.SSVM", true, true); public static final AlertType ALERT_TYPE_USAGE_SERVER_RESULT = new AlertType((short)20, "ALERT.USAGE.RESULT", true); public static final AlertType ALERT_TYPE_STORAGE_DELETE = new AlertType((short)21, "ALERT.STORAGE.DELETE", true); public static final AlertType ALERT_TYPE_UPDATE_RESOURCE_COUNT = new AlertType((short)22, "ALERT.RESOURCE.COUNT", true); public static final AlertType ALERT_TYPE_USAGE_SANITY_RESULT = new AlertType((short)23, "ALERT.USAGE.SANITY", true); public static final AlertType ALERT_TYPE_DIRECT_ATTACHED_PUBLIC_IP = new AlertType((short)24, "ALERT.NETWORK.DIRECTPUBLICIP", true); public static final AlertType ALERT_TYPE_LOCAL_STORAGE = new AlertType((short)25, "ALERT.STORAGE.LOCAL", true); - public static final AlertType ALERT_TYPE_RESOURCE_LIMIT_EXCEEDED = new AlertType((short)26, "ALERT.RESOURCE.EXCEED", true); + public static final AlertType ALERT_TYPE_RESOURCE_LIMIT_EXCEEDED = new AlertType((short)26, "ALERT.RESOURCE.EXCEED", true, true); public static final AlertType ALERT_TYPE_SYNC = new AlertType((short)27, "ALERT.TYPE.SYNC", true); - public static final AlertType ALERT_TYPE_UPLOAD_FAILED = new AlertType((short)28, "ALERT.UPLOAD.FAILED", true); - public static final AlertType ALERT_TYPE_OOBM_AUTH_ERROR = new AlertType((short)29, "ALERT.OOBM.AUTHERROR", true); - public static final AlertType ALERT_TYPE_HA_ACTION = new AlertType((short)30, "ALERT.HA.ACTION", true); - public static final AlertType ALERT_TYPE_CA_CERT = new AlertType((short)31, "ALERT.CA.CERT", true); + public static final AlertType ALERT_TYPE_UPLOAD_FAILED = new AlertType((short)28, "ALERT.UPLOAD.FAILED", true, true); + public static final AlertType ALERT_TYPE_OOBM_AUTH_ERROR = new AlertType((short)29, "ALERT.OOBM.AUTHERROR", true, true); + public static final AlertType ALERT_TYPE_HA_ACTION = new AlertType((short)30, "ALERT.HA.ACTION", true, true); + public static final AlertType ALERT_TYPE_CA_CERT = new AlertType((short)31, "ALERT.CA.CERT", true, true); public static final AlertType ALERT_TYPE_VM_SNAPSHOT = new AlertType((short)32, "ALERT.VM.SNAPSHOT", true); - public static final AlertType ALERT_TYPE_VR_PUBLIC_IFACE_MTU = new AlertType((short)32, "ALERT.VR.PUBLIC.IFACE.MTU", true); - public static final AlertType ALERT_TYPE_VR_PRIVATE_IFACE_MTU = new AlertType((short)32, "ALERT.VR.PRIVATE.IFACE.MTU", true); - public static final AlertType ALERT_TYPE_EXTENSION_PATH_NOT_READY = new AlertType((short)33, "ALERT.TYPE.EXTENSION.PATH.NOT.READY", true); - public static final AlertType ALERT_TYPE_VPN_GATEWAY_OBSOLETE_PARAMETERS = new AlertType((short)34, "ALERT.S2S.VPN.GATEWAY.OBSOLETE.PARAMETERS", true); + public static final AlertType ALERT_TYPE_VR_PUBLIC_IFACE_MTU = new AlertType((short)33, "ALERT.VR.PUBLIC.IFACE.MTU", true); + public static final AlertType ALERT_TYPE_VR_PRIVATE_IFACE_MTU = new AlertType((short)34, "ALERT.VR.PRIVATE.IFACE.MTU", true); + public static final AlertType ALERT_TYPE_EXTENSION_PATH_NOT_READY = new AlertType((short)33, "ALERT.TYPE.EXTENSION.PATH.NOT.READY", true, true); + public static final AlertType ALERT_TYPE_VPN_GATEWAY_OBSOLETE_PARAMETERS = new AlertType((short)34, "ALERT.S2S.VPN.GATEWAY.OBSOLETE.PARAMETERS", true, true); public static final AlertType ALERT_TYPE_BACKUP_STORAGE = new AlertType(Capacity.CAPACITY_TYPE_BACKUP_STORAGE, "ALERT.STORAGE.BACKUP", true); public static final AlertType ALERT_TYPE_OBJECT_STORAGE = new AlertType(Capacity.CAPACITY_TYPE_OBJECT_STORAGE, "ALERT.STORAGE.OBJECT", true); @@ -86,6 +92,10 @@ public interface AlertService { return name; } + public boolean isRepetitionAllowed() { + return repetitionAllowed; + } + private static AlertType getAlertType(short type) { for (AlertType alertType : defaultAlertTypes) { if (alertType.getType() == type) { @@ -109,7 +119,7 @@ public interface AlertService { if (defaultAlert != null && !defaultAlert.getName().equalsIgnoreCase(name)) { throw new InvalidParameterValueException("There is a default alert having type " + type + " and name " + defaultAlert.getName()); } else { - return new AlertType(type, name, false); + return new AlertType(type, name, false, false); } } } diff --git a/api/src/main/java/org/apache/cloudstack/api/APICommand.java b/api/src/main/java/org/apache/cloudstack/api/APICommand.java index c559be08116..b77649046ca 100644 --- a/api/src/main/java/org/apache/cloudstack/api/APICommand.java +++ b/api/src/main/java/org/apache/cloudstack/api/APICommand.java @@ -50,4 +50,6 @@ public @interface APICommand { RoleType[] authorized() default {}; Class[] entityType() default {}; + + String httpMethod() default ""; } diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 28169e821fe..2be0df657fe 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -282,6 +282,7 @@ public class ApiConstants { public static final String HOST = "host"; public static final String HOST_CONTROL_STATE = "hostcontrolstate"; public static final String HOSTS_MAP = "hostsmap"; + public static final String HTTP_REQUEST_TYPE = "httprequesttype"; public static final String HYPERVISOR = "hypervisor"; public static final String INLINE = "inline"; public static final String INSTANCE = "instance"; @@ -502,6 +503,7 @@ public class ApiConstants { public static final String RECONNECT = "reconnect"; public static final String RECOVER = "recover"; public static final String REPAIR = "repair"; + public static final String REPETITION_ALLOWED = "repetitionallowed"; public static final String REQUIRES_HVM = "requireshvm"; public static final String RESOURCES = "resources"; public static final String RESOURCE_COUNT = "resourcecount"; @@ -1167,6 +1169,7 @@ public class ApiConstants { public static final String OVM3_VIP = "ovm3vip"; public static final String CLEAN_UP_DETAILS = "cleanupdetails"; public static final String CLEAN_UP_EXTERNAL_DETAILS = "cleanupexternaldetails"; + public static final String CLEAN_UP_EXTRA_CONFIG = "cleanupextraconfig"; public static final String CLEAN_UP_PARAMETERS = "cleanupparameters"; public static final String VIRTUAL_SIZE = "virtualsize"; public static final String NETSCALER_CONTROLCENTER_ID = "netscalercontrolcenterid"; @@ -1217,6 +1220,7 @@ public class ApiConstants { public static final String DOCKER_REGISTRY_EMAIL = "dockerregistryemail"; public static final String ISO_NAME = "isoname"; public static final String ISO_STATE = "isostate"; + public static final String ISO_URL = "isourl"; public static final String SEMANTIC_VERSION = "semanticversion"; public static final String KUBERNETES_VERSION_ID = "kubernetesversionid"; public static final String KUBERNETES_VERSION_NAME = "kubernetesversionname"; diff --git a/api/src/main/java/org/apache/cloudstack/api/BaseUpdateTemplateOrIsoCmd.java b/api/src/main/java/org/apache/cloudstack/api/BaseUpdateTemplateOrIsoCmd.java index d16a0d49d8a..6da9db57ee3 100644 --- a/api/src/main/java/org/apache/cloudstack/api/BaseUpdateTemplateOrIsoCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/BaseUpdateTemplateOrIsoCmd.java @@ -42,7 +42,7 @@ public abstract class BaseUpdateTemplateOrIsoCmd extends BaseCmd { @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = TemplateResponse.class, required = true, description = "The ID of the image file") private Long id; - @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, description = "The name of the image file") + @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, length = 251, description = "The name of the image file") private String templateName; @Parameter(name = ApiConstants.OS_TYPE_ID, diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/domain/ListDomainsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/domain/ListDomainsCmd.java index 5c5a92c45ca..aa197804226 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/domain/ListDomainsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/domain/ListDomainsCmd.java @@ -23,7 +23,7 @@ import java.util.List; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiConstants.DomainDetails; -import org.apache.cloudstack.api.BaseListCmd; +import org.apache.cloudstack.api.BaseListTaggedResourcesCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ResponseObject.ResponseView; import org.apache.cloudstack.api.command.user.UserCmd; @@ -39,7 +39,7 @@ import com.cloud.server.ResourceTag; @APICommand(name = "listDomains", description = "Lists domains and provides detailed information for listed domains", responseObject = DomainResponse.class, responseView = ResponseView.Restricted, entityType = {Domain.class}, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) -public class ListDomainsCmd extends BaseListCmd implements UserCmd { +public class ListDomainsCmd extends BaseListTaggedResourcesCmd implements UserCmd { private static final String s_name = "listdomainsresponse"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/AddSecondaryStorageCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/AddSecondaryStorageCmd.java index 9a7eff7e2e5..585fd1b87a8 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/AddSecondaryStorageCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/AddSecondaryStorageCmd.java @@ -29,6 +29,11 @@ import org.apache.cloudstack.api.response.ZoneResponse; import com.cloud.exception.DiscoveryException; import com.cloud.storage.ImageStore; import com.cloud.user.Account; +import org.apache.commons.collections.MapUtils; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; @APICommand(name = "addSecondaryStorage", description = "Adds secondary storage.", responseObject = ImageStoreResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) @@ -44,6 +49,9 @@ public class AddSecondaryStorageCmd extends BaseCmd { @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.UUID, entityType = ZoneResponse.class, description = "The Zone ID for the secondary storage") protected Long zoneId; + @Parameter(name = ApiConstants.DETAILS, type = CommandType.MAP, description = "Details in key/value pairs using format details[i].keyname=keyvalue. Example: details[0].copytemplatesfromothersecondarystorages=true") + protected Map details; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -56,6 +64,20 @@ public class AddSecondaryStorageCmd extends BaseCmd { return zoneId; } + public Map getDetails() { + Map detailsMap = new HashMap<>(); + if (MapUtils.isNotEmpty(details)) { + Collection props = details.values(); + for (Object prop : props) { + HashMap detail = (HashMap) prop; + for (Map.Entry entry: detail.entrySet()) { + detailsMap.put(entry.getKey(),entry.getValue()); + } + } + } + return detailsMap; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @@ -68,7 +90,7 @@ public class AddSecondaryStorageCmd extends BaseCmd { @Override public void execute(){ try{ - ImageStore result = _storageService.discoverImageStore(null, getUrl(), "NFS", getZoneId(), null); + ImageStore result = _storageService.discoverImageStore(null, getUrl(), "NFS", getZoneId(), getDetails()); ImageStoreResponse storeResponse = null; if (result != null ) { storeResponse = _responseGenerator.createImageStoreResponse(result); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java index 983ed25d953..e202dfad77b 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java @@ -118,6 +118,9 @@ public class ListHostsCmd extends BaseListCmd { since = "4.21.0") private String storageAccessGroup; + @Parameter(name = ApiConstants.VERSION, type = CommandType.STRING, description = "the host version", since = "4.20.3") + private String version; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -222,6 +225,10 @@ public class ListHostsCmd extends BaseListCmd { this.storageAccessGroup = storageAccessGroup; } + public String getVersion() { + return version; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/management/ListMgmtsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/management/ListMgmtsCmd.java index ecc59f7cab8..293cb34e702 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/management/ListMgmtsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/management/ListMgmtsCmd.java @@ -45,6 +45,10 @@ public class ListMgmtsCmd extends BaseListCmd { since = "4.20.1.0") private Boolean peers; + @Parameter(name = ApiConstants.VERSION, type = CommandType.STRING, + description = "the version of the management server", since = "4.20.3") + private String version; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -61,6 +65,10 @@ public class ListMgmtsCmd extends BaseListCmd { return BooleanUtils.toBooleanDefaultIfNull(peers, false); } + public String getVersion() { + return version; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/network/UpdateNetworkOfferingCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/network/UpdateNetworkOfferingCmd.java index e3fac81a793..df9f6ad0664 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/network/UpdateNetworkOfferingCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/network/UpdateNetworkOfferingCmd.java @@ -75,6 +75,7 @@ public class UpdateNetworkOfferingCmd extends BaseCmd implements DomainAndZoneId @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.STRING, + length = 4096, description = "The ID of the containing domain(s) as comma separated string, public for public offerings") private String domainIds; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/IsAccountAllowedToCreateOfferingsWithTagsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/IsAccountAllowedToCreateOfferingsWithTagsCmd.java index fcd6b03d3e5..4b1cd2ff725 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/IsAccountAllowedToCreateOfferingsWithTagsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/IsAccountAllowedToCreateOfferingsWithTagsCmd.java @@ -26,7 +26,8 @@ import org.apache.cloudstack.api.response.AccountResponse; import org.apache.cloudstack.api.response.IsAccountAllowedToCreateOfferingsWithTagsResponse; @APICommand(name = "isAccountAllowedToCreateOfferingsWithTags", description = "Return true if the specified account is allowed to create offerings with tags.", - responseObject = IsAccountAllowedToCreateOfferingsWithTagsResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) + responseObject = IsAccountAllowedToCreateOfferingsWithTagsResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, + httpMethod = "GET") public class IsAccountAllowedToCreateOfferingsWithTagsCmd extends BaseCmd { @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = AccountResponse.class, description = "Account UUID", required = true) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/UpdateDiskOfferingCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/UpdateDiskOfferingCmd.java index 917d7ff42d8..4d48327eeb7 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/UpdateDiskOfferingCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/UpdateDiskOfferingCmd.java @@ -73,6 +73,7 @@ public class UpdateDiskOfferingCmd extends BaseCmd implements DomainAndZoneIdRes @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.STRING, description = "The ID of the containing zone(s) as comma separated string, all for all zones offerings", + length = 4096, since = "4.13") private String zoneIds; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/UpdateServiceOfferingCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/UpdateServiceOfferingCmd.java index 3a6d6639a5b..8e37499c95e 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/UpdateServiceOfferingCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/UpdateServiceOfferingCmd.java @@ -68,6 +68,7 @@ public class UpdateServiceOfferingCmd extends BaseCmd implements DomainAndZoneId @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.STRING, description = "The ID of the containing zone(s) as comma separated string, all for all zones offerings", + length = 4096, since = "4.13") private String zoneIds; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/resource/ListAlertTypesCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/resource/ListAlertTypesCmd.java index e7bfbdbc625..dcd4f2c89ef 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/resource/ListAlertTypesCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/resource/ListAlertTypesCmd.java @@ -16,7 +16,10 @@ // under the License. package org.apache.cloudstack.api.command.admin.resource; -import com.cloud.user.Account; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + import org.apache.cloudstack.alert.AlertService; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.BaseCmd; @@ -24,9 +27,7 @@ import org.apache.cloudstack.api.response.AlertResponse; import org.apache.cloudstack.api.response.AlertTypeResponse; import org.apache.cloudstack.api.response.ListResponse; -import java.util.ArrayList; -import java.util.List; -import java.util.Set; +import com.cloud.user.Account; @APICommand(name = "listAlertTypes", description = "Lists all alerts types", responseObject = AlertResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) @@ -43,7 +44,8 @@ public class ListAlertTypesCmd extends BaseCmd { ListResponse response = new ListResponse<>(); List typeResponseList = new ArrayList<>(); for (AlertService.AlertType alertType : result) { - AlertTypeResponse alertResponse = new AlertTypeResponse(alertType.getType(), alertType.getName()); + AlertTypeResponse alertResponse = new AlertTypeResponse(alertType.getType(), alertType.getName(), + alertType.isRepetitionAllowed()); alertResponse.setObjectName("alerttype"); typeResponseList.add(alertResponse); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportUnmanagedInstanceCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportUnmanagedInstanceCmd.java index 1164037dfc8..ad7e8a48b25 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportUnmanagedInstanceCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportUnmanagedInstanceCmd.java @@ -292,7 +292,7 @@ public class ImportUnmanagedInstanceCmd extends BaseAsyncCmd { @Override public long getEntityOwnerId() { - Long accountId = _accountService.finalyzeAccountId(accountName, domainId, projectId, true); + Long accountId = _accountService.finalizeAccountId(accountName, domainId, projectId, true); if (accountId == null) { Account account = CallContext.current().getCallingAccount(); if (account != null) { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmd.java index 57c3ee586d3..50f4b9c1fbe 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmd.java @@ -156,7 +156,7 @@ public class ImportVolumeCmd extends BaseAsyncCmd { @Override public long getEntityOwnerId() { - Long accountId = _accountService.finalyzeAccountId(accountName, domainId, projectId, true); + Long accountId = _accountService.finalizeAccountId(accountName, domainId, projectId, true); if (accountId == null) { return CallContext.current().getCallingAccount().getId(); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vpc/UpdateVPCOfferingCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vpc/UpdateVPCOfferingCmd.java index 300584428ea..d4565cbada2 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vpc/UpdateVPCOfferingCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vpc/UpdateVPCOfferingCmd.java @@ -61,6 +61,7 @@ public class UpdateVPCOfferingCmd extends BaseAsyncCmd implements DomainAndZoneI @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.STRING, description = "The ID of the containing zone(s) as comma separated string, all for all zones offerings", + length = 4096, since = "4.13") private String zoneIds; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/CreateAutoScaleVmProfileCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/CreateAutoScaleVmProfileCmd.java index 4ecce59fa45..db9688aa09a 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/CreateAutoScaleVmProfileCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/CreateAutoScaleVmProfileCmd.java @@ -232,7 +232,7 @@ public class CreateAutoScaleVmProfileCmd extends BaseAsyncCreateCmd { @Override public long getEntityOwnerId() { - Long accountId = _accountService.finalyzeAccountId(accountName, domainId, projectId, true); + Long accountId = _accountService.finalizeAccountId(accountName, domainId, projectId, true); if (accountId == null) { return CallContext.current().getCallingAccount().getId(); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/CreateConditionCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/CreateConditionCmd.java index 9d42706cb7d..2bb101de559 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/CreateConditionCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/CreateConditionCmd.java @@ -137,7 +137,7 @@ public class CreateConditionCmd extends BaseAsyncCreateCmd { @Override public long getEntityOwnerId() { - Long accountId = _accountService.finalyzeAccountId(accountName, domainId, projectId, true); + Long accountId = _accountService.finalizeAccountId(accountName, domainId, projectId, true); if (accountId == null) { return CallContext.current().getCallingAccount().getId(); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListBackupScheduleCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListBackupScheduleCmd.java index eed34637615..17bd06bfdd4 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListBackupScheduleCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListBackupScheduleCmd.java @@ -59,7 +59,6 @@ public class ListBackupScheduleCmd extends BaseListProjectAndAccountResourcesCmd @Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID, type = CommandType.UUID, entityType = UserVmResponse.class, - required = true, description = "ID of the Instance") private Long vmId; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/CreateBucketCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/CreateBucketCmd.java index 722556b8e2d..f1c149854d9 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/CreateBucketCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/CreateBucketCmd.java @@ -150,7 +150,7 @@ public class CreateBucketCmd extends BaseAsyncCreateCmd implements UserCmd { @Override public long getEntityOwnerId() { - Long accountId = _accountService.finalyzeAccountId(accountName, domainId, projectId, true); + Long accountId = _accountService.finalizeAccountId(accountName, domainId, projectId, true); if (accountId == null) { return CallContext.current().getCallingAccount().getId(); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/config/ListCapabilitiesCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/config/ListCapabilitiesCmd.java index 94b6062b621..2cb64070950 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/config/ListCapabilitiesCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/config/ListCapabilitiesCmd.java @@ -63,6 +63,7 @@ public class ListCapabilitiesCmd extends BaseCmd { response.setDiskOffMaxSize((Long)capabilities.get("customDiskOffMaxSize")); response.setRegionSecondaryEnabled((Boolean)capabilities.get("regionSecondaryEnabled")); response.setKVMSnapshotEnabled((Boolean)capabilities.get("KVMSnapshotEnabled")); + response.setSnapshotShowChainSize((Boolean)capabilities.get("SnapshotShowChainSize")); response.setAllowUserViewDestroyedVM((Boolean)capabilities.get("allowUserViewDestroyedVM")); response.setAllowUserExpungeRecoverVM((Boolean)capabilities.get("allowUserExpungeRecoverVM")); response.setAllowUserExpungeRecoverVolume((Boolean)capabilities.get("allowUserExpungeRecoverVolume")); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/iso/GetUploadParamsForIsoCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/iso/GetUploadParamsForIsoCmd.java index 564ebb20b75..43cdf09a89c 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/iso/GetUploadParamsForIsoCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/iso/GetUploadParamsForIsoCmd.java @@ -158,7 +158,7 @@ public class GetUploadParamsForIsoCmd extends AbstractGetUploadParamsCmd { @Override public long getEntityOwnerId() { - Long accountId = _accountService.finalyzeAccountId(getAccountName(), getDomainId(), getProjectId(), true); + Long accountId = _accountService.finalizeAccountId(getAccountName(), getDomainId(), getProjectId(), true); if (accountId == null) { return CallContext.current().getCallingAccount().getId(); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/iso/RegisterIsoCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/iso/RegisterIsoCmd.java index 860931aa248..1c57e902e22 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/iso/RegisterIsoCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/iso/RegisterIsoCmd.java @@ -70,7 +70,7 @@ public class RegisterIsoCmd extends BaseCmd implements UserCmd { @Parameter(name = ApiConstants.IS_EXTRACTABLE, type = CommandType.BOOLEAN, description = "True if the ISO or its derivatives are extractable; default is false") private Boolean extractable; - @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, required = true, description = "The name of the ISO") + @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, required = true, length = 251, description = "The name of the ISO") private String isoName; @Parameter(name = ApiConstants.OS_TYPE_ID, @@ -254,7 +254,7 @@ public class RegisterIsoCmd extends BaseCmd implements UserCmd { @Override public long getEntityOwnerId() { - Long accountId = _accountService.finalyzeAccountId(accountName, domainId, projectId, true); + Long accountId = _accountService.finalizeAccountId(accountName, domainId, projectId, true); if (accountId == null) { return CallContext.current().getCallingAccount().getId(); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkACLCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkACLCmd.java index a6cd724f2d1..087e31c0882 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkACLCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkACLCmd.java @@ -58,7 +58,7 @@ public class CreateNetworkACLCmd extends BaseAsyncCreateCmd { private Integer publicEndPort; @Parameter(name = ApiConstants.CIDR_LIST, type = CommandType.LIST, collectionType = CommandType.STRING, description = "The CIDR list to allow traffic from/to. Multiple entries must be separated by a single comma character (,).") - private List cidrlist; + private List cidrList; @Parameter(name = ApiConstants.ICMP_TYPE, type = CommandType.INTEGER, description = "Type of the ICMP message being sent") private Integer icmpType; @@ -118,8 +118,8 @@ public class CreateNetworkACLCmd extends BaseAsyncCreateCmd { } public List getSourceCidrList() { - if (cidrlist != null) { - return cidrlist; + if (cidrList != null) { + return cidrList; } else { List oneCidrList = new ArrayList(); oneCidrList.add(NetUtils.ALL_IP4_CIDRS); @@ -238,6 +238,30 @@ public class CreateNetworkACLCmd extends BaseAsyncCreateCmd { return reason; } + public void setCidrList(List cidrList) { + this.cidrList = cidrList; + } + + public void setIcmpType(Integer icmpType) { + this.icmpType = icmpType; + } + + public void setIcmpCode(Integer icmpCode) { + this.icmpCode = icmpCode; + } + + public void setNumber(Integer number) { + this.number = number; + } + + public void setDisplay(Boolean display) { + this.display = display; + } + + public void setReason(String reason) { + this.reason = reason; + } + @Override public void create() { NetworkACLItem result = _networkACLService.createNetworkACLItem(this); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkCmd.java index 35fec1d6b3e..cbf6df081b3 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkCmd.java @@ -417,7 +417,7 @@ public class CreateNetworkCmd extends BaseCmd implements UserCmd { @Override public long getEntityOwnerId() { - Long accountId = _accountService.finalyzeAccountId(accountName, domainId, projectId, true); + Long accountId = _accountService.finalizeAccountId(accountName, domainId, projectId, true); if (accountId == null) { return CallContext.current().getCallingAccount().getId(); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/network/ImportNetworkACLCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/network/ImportNetworkACLCmd.java new file mode 100644 index 00000000000..daf3633ce2a --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/network/ImportNetworkACLCmd.java @@ -0,0 +1,132 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.network; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.NetworkACLItemResponse; +import org.apache.cloudstack.api.response.NetworkACLResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.commons.collections.MapUtils; + +import com.cloud.event.EventTypes; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.network.vpc.NetworkACLItem; +import com.cloud.user.Account; + +@APICommand(name = "importNetworkACL", description = "Imports Network ACL rules.", + responseObject = NetworkACLItemResponse.class, + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, + since = "4.22.1") +public class ImportNetworkACLCmd extends BaseAsyncCmd { + + // /////////////////////////////////////////////////// + // ////////////// API parameters ///////////////////// + // /////////////////////////////////////////////////// + + @Parameter( + name = ApiConstants.ACL_ID, + type = CommandType.UUID, + entityType = NetworkACLResponse.class, + required = true, + description = "The ID of the Network ACL to which the rules will be imported" + ) + private Long aclId; + + @Parameter(name = ApiConstants.RULES, type = CommandType.MAP, required = true, + description = "Rules param list, id and protocol are must. Invalid rules will be discarded. Example: " + + "rules[0].id=101&rules[0].protocol=tcp&rules[0].traffictype=ingress&rules[0].state=active&rules[0].cidrlist=192.168.1.0/24" + + "&rules[0].tags=web&rules[0].aclid=acl-001&rules[0].aclname=web-acl&rules[0].number=1&rules[0].action=allow&rules[0].fordisplay=true" + + "&rules[0].description=allow%20web%20traffic&rules[1].id=102&rules[1].protocol=udp&rules[1].traffictype=egress&rules[1].state=enabled" + + "&rules[1].cidrlist=10.0.0.0/8&rules[1].tags=db&rules[1].aclid=acl-002&rules[1].aclname=db-acl&rules[1].number=2&rules[1].action=deny" + + "&rules[1].fordisplay=false&rules[1].description=deny%20database%20traffic") + private Map rules; + + + // /////////////////////////////////////////////////// + // ///////////////// Accessors /////////////////////// + // /////////////////////////////////////////////////// + + // Returns map, corresponds to a rule with the details in the keys: + // id, protocol, startport, endport, traffictype, state, cidrlist, tags, aclid, aclname, number, action, fordisplay, description + public Map getRules() { + return rules; + } + + public Long getAclId() { + return aclId; + } + + // /////////////////////////////////////////////////// + // ///////////// API Implementation/////////////////// + // /////////////////////////////////////////////////// + + + @Override + public void execute() throws ResourceUnavailableException { + validateParams(); + List importedRules = _networkACLService.importNetworkACLRules(this); + ListResponse response = new ListResponse<>(); + List aclResponse = new ArrayList<>(); + for (NetworkACLItem acl : importedRules) { + NetworkACLItemResponse ruleData = _responseGenerator.createNetworkACLItemResponse(acl); + aclResponse.add(ruleData); + } + response.setResponses(aclResponse, importedRules.size()); + response.setResponseName(getCommandName()); + setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + Account account = CallContext.current().getCallingAccount(); + if (account != null) { + return account.getId(); + } + return Account.ACCOUNT_ID_SYSTEM; + } + + @Override + public String getEventType() { + return EventTypes.EVENT_NETWORK_ACL_IMPORT; + } + + @Override + public String getEventDescription() { + return "Importing ACL rules for ACL ID: " + getAclId(); + } + + + private void validateParams() { + if(MapUtils.isEmpty(rules)) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Rules parameter is empty or null"); + } + + if (getAclId() == null || _networkACLService.getNetworkACL(getAclId()) == null) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Unable to find Network ACL with provided ACL ID"); + } + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/region/ha/gslb/CreateGlobalLoadBalancerRuleCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/region/ha/gslb/CreateGlobalLoadBalancerRuleCmd.java index df9b6b31865..20b227b831c 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/region/ha/gslb/CreateGlobalLoadBalancerRuleCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/region/ha/gslb/CreateGlobalLoadBalancerRuleCmd.java @@ -180,7 +180,7 @@ public class CreateGlobalLoadBalancerRuleCmd extends BaseAsyncCreateCmd { @Override public long getEntityOwnerId() { - Long accountId = _accountService.finalyzeAccountId(accountName, domainId, null, true); + Long accountId = _accountService.finalizeAccountId(accountName, domainId, null, true); if (accountId == null) { return CallContext.current().getCallingAccount().getId(); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/resource/ListResourceLimitsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/resource/ListResourceLimitsCmd.java index 7bae74c73a4..4bda2361709 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/resource/ListResourceLimitsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/resource/ListResourceLimitsCmd.java @@ -103,7 +103,7 @@ public class ListResourceLimitsCmd extends BaseListProjectAndAccountResourcesCmd @Override public void execute() { List result = - _resourceLimitService.searchForLimits(id, _accountService.finalyzeAccountId(this.getAccountName(), this.getDomainId(), this.getProjectId(), false), this.getDomainId(), + _resourceLimitService.searchForLimits(id, _accountService.finalizeAccountId(this.getAccountName(), this.getDomainId(), this.getProjectId(), false), this.getDomainId(), getResourceTypeEnum(), getTag(), this.getStartIndex(), this.getPageSizeVal()); ListResponse response = new ListResponse(); List limitResponses = new ArrayList(); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/resource/UpdateResourceCountCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/resource/UpdateResourceCountCmd.java index 123b0e48a74..d43bb29e9d2 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/resource/UpdateResourceCountCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/resource/UpdateResourceCountCmd.java @@ -127,7 +127,7 @@ public class UpdateResourceCountCmd extends BaseCmd { @Override public void execute() { List result = - _resourceLimitService.recalculateResourceCount(_accountService.finalyzeAccountId(accountName, domainId, projectId, true), getDomainId(), getResourceType(), getTag()); + _resourceLimitService.recalculateResourceCount(_accountService.finalizeAccountId(accountName, domainId, projectId, true), getDomainId(), getResourceType(), getTag()); if ((result != null) && (result.size() > 0)) { ListResponse response = new ListResponse(); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/resource/UpdateResourceLimitCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/resource/UpdateResourceLimitCmd.java index 3678e885a6e..f88ef9678e3 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/resource/UpdateResourceLimitCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/resource/UpdateResourceLimitCmd.java @@ -100,7 +100,7 @@ public class UpdateResourceLimitCmd extends BaseCmd { @Override public long getEntityOwnerId() { - Long accountId = _accountService.finalyzeAccountId(accountName, domainId, projectId, true); + Long accountId = _accountService.finalizeAccountId(accountName, domainId, projectId, true); if (accountId == null) { return CallContext.current().getCallingAccount().getId(); } @@ -110,7 +110,7 @@ public class UpdateResourceLimitCmd extends BaseCmd { @Override public void execute() { - ResourceLimit result = _resourceLimitService.updateResourceLimit(_accountService.finalyzeAccountId(accountName, domainId, projectId, true), getDomainId(), resourceType, max, getTag()); + ResourceLimit result = _resourceLimitService.updateResourceLimit(_accountService.finalizeAccountId(accountName, domainId, projectId, true), getDomainId(), resourceType, max, getTag()); if (result != null || (result == null && max != null && max.longValue() == -1L)) { ResourceLimitResponse response = _responseGenerator.createResourceLimitResponse(result); response.setResponseName(getCommandName()); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/securitygroup/AuthorizeSecurityGroupEgressCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/securitygroup/AuthorizeSecurityGroupEgressCmd.java index 13b09c7e19a..7d0004c8e5d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/securitygroup/AuthorizeSecurityGroupEgressCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/securitygroup/AuthorizeSecurityGroupEgressCmd.java @@ -166,7 +166,7 @@ public class AuthorizeSecurityGroupEgressCmd extends BaseAsyncCmd { @Override public long getEntityOwnerId() { - Long accountId = _accountService.finalyzeAccountId(accountName, domainId, projectId, true); + Long accountId = _accountService.finalizeAccountId(accountName, domainId, projectId, true); if (accountId == null) { return CallContext.current().getCallingAccount().getId(); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/securitygroup/AuthorizeSecurityGroupIngressCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/securitygroup/AuthorizeSecurityGroupIngressCmd.java index 07a4df9eb5d..d7a95d8204e 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/securitygroup/AuthorizeSecurityGroupIngressCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/securitygroup/AuthorizeSecurityGroupIngressCmd.java @@ -166,7 +166,7 @@ public class AuthorizeSecurityGroupIngressCmd extends BaseAsyncCmd { @Override public long getEntityOwnerId() { - Long accountId = _accountService.finalyzeAccountId(accountName, domainId, projectId, true); + Long accountId = _accountService.finalizeAccountId(accountName, domainId, projectId, true); if (accountId == null) { return CallContext.current().getCallingAccount().getId(); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/securitygroup/DeleteSecurityGroupCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/securitygroup/DeleteSecurityGroupCmd.java index 0636f03c92a..1882d80c1c1 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/securitygroup/DeleteSecurityGroupCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/securitygroup/DeleteSecurityGroupCmd.java @@ -103,7 +103,7 @@ public class DeleteSecurityGroupCmd extends BaseCmd { @Override public long getEntityOwnerId() { - Long accountId = _accountService.finalyzeAccountId(accountName, domainId, projectId, true); + Long accountId = _accountService.finalizeAccountId(accountName, domainId, projectId, true); if (accountId == null) { return CallContext.current().getCallingAccount().getId(); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java index f78112d679f..b2b1da91abe 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java @@ -274,8 +274,7 @@ public class CreateSnapshotCmd extends BaseAsyncCreateCmd { } public Snapshot.LocationType getLocationType() { - - if (Snapshot.LocationType.values() == null || Snapshot.LocationType.values().length == 0 || locationType == null) { + if (locationType == null) { return null; } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/ssh/CreateSSHKeyPairCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/ssh/CreateSSHKeyPairCmd.java index a55b2059302..1b79c11644f 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/ssh/CreateSSHKeyPairCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/ssh/CreateSSHKeyPairCmd.java @@ -77,7 +77,7 @@ public class CreateSSHKeyPairCmd extends BaseCmd { ///////////////////////////////////////////////////// @Override public long getEntityOwnerId() { - Long accountId = _accountService.finalyzeAccountId(accountName, domainId, projectId, true); + Long accountId = _accountService.finalizeAccountId(accountName, domainId, projectId, true); if (accountId == null) { return CallContext.current().getCallingAccount().getId(); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/ssh/RegisterSSHKeyPairCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/ssh/RegisterSSHKeyPairCmd.java index 36c708ea111..f7af86d0835 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/ssh/RegisterSSHKeyPairCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/ssh/RegisterSSHKeyPairCmd.java @@ -85,7 +85,7 @@ public class RegisterSSHKeyPairCmd extends BaseCmd { @Override public long getEntityOwnerId() { - Long accountId = _accountService.finalyzeAccountId(accountName, domainId, projectId, true); + Long accountId = _accountService.finalizeAccountId(accountName, domainId, projectId, true); if (accountId == null) { return CallContext.current().getCallingAccount().getId(); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/storage/sharedfs/CreateSharedFSCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/storage/sharedfs/CreateSharedFSCmd.java index ddaa31612a8..595b611b5c0 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/storage/sharedfs/CreateSharedFSCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/storage/sharedfs/CreateSharedFSCmd.java @@ -230,7 +230,7 @@ public class CreateSharedFSCmd extends BaseAsyncCreateCmd implements UserCmd { } @Override public long getEntityOwnerId() { - Long accountId = _accountService.finalyzeAccountId(accountName, domainId, projectId, true); + Long accountId = _accountService.finalizeAccountId(accountName, domainId, projectId, true); if (accountId == null) { return CallContext.current().getCallingAccount().getId(); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/template/CreateTemplateCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/template/CreateTemplateCmd.java index 3720e2c89f1..76fadb7853b 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/template/CreateTemplateCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/template/CreateTemplateCmd.java @@ -354,7 +354,7 @@ public class CreateTemplateCmd extends BaseAsyncCreateCmd implements UserCmd { private Long findAccountIdToUse(Account callingAccount) { Long accountIdToUse = null; try { - accountIdToUse = _accountService.finalyzeAccountId(accountName, domainId, projectId, true); + accountIdToUse = _accountService.finalizeAccountId(accountName, domainId, projectId, true); } catch (InvalidParameterValueException | PermissionDeniedException ex) { logger.error("Unable to find accountId associated with accountName={} and domainId={} or projectId={}" + ", using callingAccountId={}", accountName, domainId, projectId, callingAccount.getUuid()); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/template/GetUploadParamsForTemplateCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/template/GetUploadParamsForTemplateCmd.java index 5058790d1f4..e6e178baada 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/template/GetUploadParamsForTemplateCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/template/GetUploadParamsForTemplateCmd.java @@ -223,7 +223,7 @@ public class GetUploadParamsForTemplateCmd extends AbstractGetUploadParamsCmd { @Override public long getEntityOwnerId() { - Long accountId = _accountService.finalyzeAccountId(getAccountName(), getDomainId(), getProjectId(), true); + Long accountId = _accountService.finalizeAccountId(getAccountName(), getDomainId(), getProjectId(), true); if (accountId == null) { return CallContext.current().getCallingAccount().getId(); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java index b8a0a0c6756..49992ac6661 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java @@ -344,7 +344,7 @@ public class RegisterTemplateCmd extends BaseCmd implements UserCmd { @Override public long getEntityOwnerId() { - Long accountId = _accountService.finalyzeAccountId(accountName, domainId, projectId, true); + Long accountId = _accountService.finalizeAccountId(accountName, domainId, projectId, true); if (accountId == null) { return CallContext.current().getCallingAccount().getId(); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/RegisterCniConfigurationCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/RegisterCniConfigurationCmd.java index eb80da3be05..3f1de41eab8 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/RegisterCniConfigurationCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/RegisterCniConfigurationCmd.java @@ -67,7 +67,7 @@ public class RegisterCniConfigurationCmd extends BaseRegisterUserDataCmd { @Override public long getEntityOwnerId() { - Long accountId = _accountService.finalyzeAccountId(getAccountName(), getDomainId(), getProjectId(), true); + Long accountId = _accountService.finalizeAccountId(getAccountName(), getDomainId(), getProjectId(), true); if (accountId == null) { return CallContext.current().getCallingAccount().getId(); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/RegisterUserDataCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/RegisterUserDataCmd.java index cbbe7674814..d99f2fd066d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/RegisterUserDataCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/RegisterUserDataCmd.java @@ -70,7 +70,7 @@ public class RegisterUserDataCmd extends BaseRegisterUserDataCmd { @Override public long getEntityOwnerId() { - Long accountId = _accountService.finalyzeAccountId(getAccountName(), getDomainId(), getProjectId(), true); + Long accountId = _accountService.finalizeAccountId(getAccountName(), getDomainId(), getProjectId(), true); if (accountId == null) { return CallContext.current().getCallingAccount().getId(); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/BaseDeployVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/BaseDeployVMCmd.java index a72fdee3a58..07c11b21107 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/BaseDeployVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/BaseDeployVMCmd.java @@ -808,7 +808,7 @@ public abstract class BaseDeployVMCmd extends BaseAsyncCreateCustomIdCmd impleme @Override public long getEntityOwnerId() { - Long accountId = _accountService.finalyzeAccountId(accountName, domainId, projectId, true); + Long accountId = _accountService.finalizeAccountId(accountName, domainId, projectId, true); if (accountId == null) { return CallContext.current().getCallingAccount().getId(); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java index 06272cadae2..528a8b0c735 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java @@ -19,6 +19,7 @@ package org.apache.cloudstack.api.command.user.vm; import java.util.Objects; import java.util.stream.Stream; +import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.api.ACL; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; @@ -39,7 +40,6 @@ import com.cloud.exception.InsufficientServerCapacityException; import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; import com.cloud.uservm.UserVm; -import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VirtualMachine; @APICommand(name = "deployVirtualMachine", description = "Creates and automatically starts an Instance based on a service offering, disk offering, and Template.", responseObject = UserVmResponse.class, responseView = ResponseView.Restricted, entityType = {VirtualMachine.class}, diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVnfApplianceCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVnfApplianceCmd.java index 4d50dd9c39b..92ddfd5b235 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVnfApplianceCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVnfApplianceCmd.java @@ -43,7 +43,7 @@ import java.util.List; public class DeployVnfApplianceCmd extends DeployVMCmd implements UserCmd { @Parameter(name = ApiConstants.VNF_CONFIGURE_MANAGEMENT, type = CommandType.BOOLEAN, required = false, - description = "True by default, security group or network rules (source nat and firewall rules) will be configured for VNF management interfaces. False otherwise. " + + description = "False by default, security group or network rules (source nat and firewall rules) will be configured for VNF management interfaces. True otherwise. " + "Network rules are configured if management network is an isolated network or shared network with security groups.") private Boolean vnfConfigureManagement; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/UpdateVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/UpdateVMCmd.java index ddc2c06eb09..115b37d61d1 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/UpdateVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/UpdateVMCmd.java @@ -21,6 +21,7 @@ import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.ResourceUnavailableException; import com.cloud.user.Account; import com.cloud.uservm.UserVm; +import com.cloud.utils.StringUtils; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.net.Dhcp; import com.cloud.vm.VirtualMachine; @@ -44,7 +45,6 @@ import org.apache.cloudstack.api.response.UserVmResponse; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.vm.lease.VMLeaseManager; import org.apache.commons.lang3.EnumUtils; -import org.apache.commons.lang3.StringUtils; import java.util.Collection; import java.util.HashMap; @@ -163,11 +163,20 @@ public class UpdateVMCmd extends BaseCustomIdCmd implements SecurityGroupAction, description = "Lease expiry action, valid values are STOP and DESTROY") private String leaseExpiryAction; + @Parameter(name = ApiConstants.CLEAN_UP_EXTRA_CONFIG, type = CommandType.BOOLEAN, since = "4.23.0", + description = "Optional boolean field, which indicates if extraconfig for the instance should be " + + "cleaned up or not (If set to true, extraconfig removed for this instance, extraconfig field " + + "ignored; if false or not set, no action)") + private Boolean cleanupExtraConfig; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// public String getDisplayName() { + if (StringUtils.isBlank(displayName)) { + displayName = name; + } return displayName; } @@ -271,14 +280,18 @@ public class UpdateVMCmd extends BaseCustomIdCmd implements SecurityGroupAction, return extraConfig; } - ///////////////////////////////////////////////////// - /////////////// API Implementation/////////////////// - ///////////////////////////////////////////////////// - public Long getOsTypeId() { return osTypeId; } + public boolean isCleanupExtraConfig() { + return Boolean.TRUE.equals(cleanupExtraConfig); + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + @Override public String getCommandName() { return s_name; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vmgroup/CreateVMGroupCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vmgroup/CreateVMGroupCmd.java index a142ffa2d49..12f534e1a22 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vmgroup/CreateVMGroupCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vmgroup/CreateVMGroupCmd.java @@ -82,7 +82,7 @@ public class CreateVMGroupCmd extends BaseCmd { @Override public long getEntityOwnerId() { - Long accountId = _accountService.finalyzeAccountId(accountName, domainId, projectId, true); + Long accountId = _accountService.finalizeAccountId(accountName, domainId, projectId, true); if (accountId == null) { return CallContext.current().getCallingAccount().getId(); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/CreateVolumeCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/CreateVolumeCmd.java index 27b592aa8f1..5938bdb810f 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/CreateVolumeCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/CreateVolumeCmd.java @@ -188,7 +188,7 @@ public class CreateVolumeCmd extends BaseAsyncCreateCustomIdCmd implements UserC @Override public long getEntityOwnerId() { - Long accountId = _accountService.finalyzeAccountId(accountName, domainId, projectId, true); + Long accountId = _accountService.finalizeAccountId(accountName, domainId, projectId, true); if (accountId == null) { return CallContext.current().getCallingAccount().getId(); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/GetUploadParamsForVolumeCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/GetUploadParamsForVolumeCmd.java index 0020fe4021e..1e3b2e46077 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/GetUploadParamsForVolumeCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/GetUploadParamsForVolumeCmd.java @@ -72,7 +72,7 @@ public class GetUploadParamsForVolumeCmd extends AbstractGetUploadParamsCmd { @Override public long getEntityOwnerId() { - Long accountId = _accountService.finalyzeAccountId(getAccountName(), getDomainId(), getProjectId(), true); + Long accountId = _accountService.finalizeAccountId(getAccountName(), getDomainId(), getProjectId(), true); if (accountId == null) { return CallContext.current().getCallingAccount().getId(); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/UploadVolumeCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/UploadVolumeCmd.java index 3d23a631722..81077deff65 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/UploadVolumeCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/UploadVolumeCmd.java @@ -159,7 +159,7 @@ public class UploadVolumeCmd extends BaseAsyncCmd implements UserCmd { @Override public long getEntityOwnerId() { - Long accountId = _accountService.finalyzeAccountId(accountName, domainId, projectId, true); + Long accountId = _accountService.finalizeAccountId(accountName, domainId, projectId, true); if (accountId == null) { return CallContext.current().getCallingAccount().getId(); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/CreateVPCCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/CreateVPCCmd.java index a91d3bad410..2adbbd66408 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/CreateVPCCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/CreateVPCCmd.java @@ -273,7 +273,7 @@ public class CreateVPCCmd extends BaseAsyncCreateCmd implements UserCmd { @Override public long getEntityOwnerId() { - Long accountId = _accountService.finalyzeAccountId(accountName, domainId, projectId, true); + Long accountId = _accountService.finalizeAccountId(accountName, domainId, projectId, true); if (accountId == null) { return CallContext.current().getCallingAccount().getId(); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vpn/AddVpnUserCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vpn/AddVpnUserCmd.java index b00b932258a..78cd9a3ac7e 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vpn/AddVpnUserCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vpn/AddVpnUserCmd.java @@ -88,7 +88,7 @@ public class AddVpnUserCmd extends BaseAsyncCreateCmd { @Override public long getEntityOwnerId() { - Long accountId = _accountService.finalyzeAccountId(accountName, domainId, projectId, true); + Long accountId = _accountService.finalizeAccountId(accountName, domainId, projectId, true); if (accountId == null) { return CallContext.current().getCallingAccount().getId(); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vpn/CreateVpnCustomerGatewayCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vpn/CreateVpnCustomerGatewayCmd.java index ef0e2354495..0da813eb486 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vpn/CreateVpnCustomerGatewayCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vpn/CreateVpnCustomerGatewayCmd.java @@ -167,7 +167,7 @@ public class CreateVpnCustomerGatewayCmd extends BaseAsyncCreateCmd { @Override public long getEntityOwnerId() { - Long accountId = _accountService.finalyzeAccountId(accountName, domainId, projectId, true); + Long accountId = _accountService.finalizeAccountId(accountName, domainId, projectId, true); if (accountId == null) { accountId = CallContext.current().getCallingAccount().getId(); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vpn/RemoveVpnUserCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vpn/RemoveVpnUserCmd.java index aff87105f9c..a18619c8949 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vpn/RemoveVpnUserCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vpn/RemoveVpnUserCmd.java @@ -82,7 +82,7 @@ public class RemoveVpnUserCmd extends BaseAsyncCmd { @Override public long getEntityOwnerId() { - Long accountId = _accountService.finalyzeAccountId(accountName, domainId, projectId, true); + Long accountId = _accountService.finalizeAccountId(accountName, domainId, projectId, true); if (accountId == null) { return CallContext.current().getCallingAccount().getId(); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vpn/ResetVpnConnectionCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vpn/ResetVpnConnectionCmd.java index b6e29e66ff4..f681c8cce18 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vpn/ResetVpnConnectionCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vpn/ResetVpnConnectionCmd.java @@ -74,7 +74,7 @@ public class ResetVpnConnectionCmd extends BaseAsyncCmd { @Override public long getEntityOwnerId() { - Long accountId = _accountService.finalyzeAccountId(accountName, domainId, null, true); + Long accountId = _accountService.finalizeAccountId(accountName, domainId, null, true); if (accountId == null) { return CallContext.current().getCallingAccount().getId(); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vpn/UpdateVpnCustomerGatewayCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vpn/UpdateVpnCustomerGatewayCmd.java index edd168f0837..56aa8b2cd16 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vpn/UpdateVpnCustomerGatewayCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vpn/UpdateVpnCustomerGatewayCmd.java @@ -161,7 +161,7 @@ public class UpdateVpnCustomerGatewayCmd extends BaseAsyncCmd { @Override public long getEntityOwnerId() { - Long accountId = _accountService.finalyzeAccountId(accountName, domainId, null, true); + Long accountId = _accountService.finalizeAccountId(accountName, domainId, null, true); if (accountId == null) { accountId = CallContext.current().getCallingAccount().getId(); } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/AlertTypeResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/AlertTypeResponse.java index 3f91cde0178..e8c3cf6c4ac 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/AlertTypeResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/AlertTypeResponse.java @@ -16,11 +16,12 @@ // under the License. package org.apache.cloudstack.api.response; -import com.cloud.serializer.Param; -import com.google.gson.annotations.SerializedName; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseResponse; +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; + public class AlertTypeResponse extends BaseResponse { @SerializedName("alerttypeid") @@ -31,6 +32,10 @@ public class AlertTypeResponse extends BaseResponse { @Param(description = "description of alert type") private String name; + @SerializedName(ApiConstants.REPETITION_ALLOWED) + @Param(description = "Whether repetitive alerts allowed for the alert type", since = "4.22.0") + private boolean repetitionAllowed = true; + public String getName() { return name; } @@ -47,9 +52,10 @@ public class AlertTypeResponse extends BaseResponse { this.alertType = alertType; } - public AlertTypeResponse(short alertType, String name) { + public AlertTypeResponse(short alertType, String name, boolean repetitionAllowed) { this.alertType = alertType; this.name = name; + this.repetitionAllowed = repetitionAllowed; setObjectName("alerttype"); } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/CapabilitiesResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/CapabilitiesResponse.java index 81621696280..7ef627ec33c 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/CapabilitiesResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/CapabilitiesResponse.java @@ -75,6 +75,10 @@ public class CapabilitiesResponse extends BaseResponse { @Param(description = "True if Snapshot is supported for KVM host, false otherwise") private boolean kvmSnapshotEnabled; + @SerializedName("snapshotshowchainsize") + @Param(description = "True to show the parent and chain size (sum of physical size of snapshot and all its parents) for incremental snapshots", since = "4.22.1") + private boolean snapshotShowChainSize; + @SerializedName("apilimitmax") @Param(description = "Max allowed number of api requests within the specified interval") private Integer apiLimitMax; @@ -203,6 +207,10 @@ public class CapabilitiesResponse extends BaseResponse { this.kvmSnapshotEnabled = kvmSnapshotEnabled; } + public void setSnapshotShowChainSize(boolean snapshotShowChainSize) { + this.snapshotShowChainSize = snapshotShowChainSize; + } + public void setApiLimitInterval(Integer apiLimitInterval) { this.apiLimitInterval = apiLimitInterval; } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/DomainResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/DomainResponse.java index e018b1a0f72..453c6b229e9 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/DomainResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/DomainResponse.java @@ -16,21 +16,21 @@ // under the License. package org.apache.cloudstack.api.response; -import com.google.gson.annotations.SerializedName; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Set; import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.BaseResponseWithAnnotations; +import org.apache.cloudstack.api.BaseResponseWithTagInformation; import org.apache.cloudstack.api.EntityReference; import com.cloud.domain.Domain; import com.cloud.serializer.Param; - -import java.util.Date; -import java.util.List; -import java.util.Map; +import com.google.gson.annotations.SerializedName; @EntityReference(value = Domain.class) -public class DomainResponse extends BaseResponseWithAnnotations implements ResourceLimitAndCountResponse, SetResourceIconResponse { +public class DomainResponse extends BaseResponseWithTagInformation implements ResourceLimitAndCountResponse, SetResourceIconResponse { @SerializedName(ApiConstants.ID) @Param(description = "The ID of the domain") private String id; @@ -589,4 +589,8 @@ public class DomainResponse extends BaseResponseWithAnnotations implements Resou public void setTaggedResourceLimitsAndCounts(List taggedResourceLimitsAndCounts) { this.taggedResources = taggedResourceLimitsAndCounts; } + + public void setTags(Set tags) { + this.tags = tags; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/SnapshotResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/SnapshotResponse.java index 827a55b1875..3db6fd87ed5 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/SnapshotResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/SnapshotResponse.java @@ -155,6 +155,14 @@ public class SnapshotResponse extends BaseResponseWithTagInformation implements @Param(description = "download progress of a snapshot", since = "4.19.0") private Map downloadDetails; + @SerializedName("parent") + @Param(description = "The parent ID of the Snapshot", since = "4.22.1") + private String parent; + + @SerializedName("parentname") + @Param(description = "The parent name of the Snapshot", since = "4.22.1") + private String parentName; + public SnapshotResponse() { tags = new LinkedHashSet(); } @@ -313,4 +321,12 @@ public class SnapshotResponse extends BaseResponseWithTagInformation implements public void setDownloadDetails(Map downloadDetails) { this.downloadDetails = downloadDetails; } + + public void setParent(String parent) { + this.parent = parent; + } + + public void setParentName(String parentName) { + this.parentName = parentName; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/SystemVmResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/SystemVmResponse.java index e69f6366742..a3ed88c2735 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/SystemVmResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/SystemVmResponse.java @@ -126,6 +126,10 @@ public class SystemVmResponse extends BaseResponseWithAnnotations { @Param(description = "The public netmask for the System VM") private String publicNetmask; + @SerializedName("storageip") + @Param(description = "the ip address for the system VM on the storage network") + private String storageIp; + @SerializedName("templateid") @Param(description = "The Template ID for the System VM") private String templateId; @@ -355,6 +359,14 @@ public class SystemVmResponse extends BaseResponseWithAnnotations { this.publicNetmask = publicNetmask; } + public String getStorageIp() { + return storageIp; + } + + public void setStorageIp(String storageIp) { + this.storageIp = storageIp; + } + public String getTemplateId() { return templateId; } diff --git a/api/src/main/java/org/apache/cloudstack/query/QueryService.java b/api/src/main/java/org/apache/cloudstack/query/QueryService.java index 3c9110595a0..5cd67ffe9ba 100644 --- a/api/src/main/java/org/apache/cloudstack/query/QueryService.java +++ b/api/src/main/java/org/apache/cloudstack/query/QueryService.java @@ -120,7 +120,7 @@ public interface QueryService { ConfigKey UserVMReadOnlyDetails = new ConfigKey<>(String.class, "user.vm.readonly.details", "Advanced", "dataDiskController, rootDiskController", - "List of read-only VM settings/details as comma separated string", true, ConfigKey.Scope.Global, null, null, null, null, null, ConfigKey.Kind.CSV, null); + "List of read-only VM settings/details as comma separated string", true, ConfigKey.Scope.Global, null, null, null, null, null, ConfigKey.Kind.CSV, null, ""); ConfigKey SortKeyAscending = new ConfigKey<>("Advanced", Boolean.class, "sortkey.algorithm", "true", "Sort algorithm - ascending or descending - to use. For entities that use sort key(template, disk offering, service offering, " + diff --git a/api/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateManager.java b/api/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateManager.java index 6571346ad65..3df59811561 100644 --- a/api/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateManager.java +++ b/api/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateManager.java @@ -29,6 +29,7 @@ import org.apache.cloudstack.api.command.user.template.UpdateVnfTemplateCmd; import org.apache.cloudstack.api.command.user.vm.DeployVnfApplianceCmd; import org.apache.cloudstack.framework.config.ConfigKey; import java.util.List; +import java.util.Map; public interface VnfTemplateManager { @@ -42,11 +43,12 @@ public interface VnfTemplateManager { void updateVnfTemplate(long templateId, UpdateVnfTemplateCmd cmd); - void validateVnfApplianceNics(VirtualMachineTemplate template, List networkIds); + void validateVnfApplianceNics(VirtualMachineTemplate template, List networkIds, Map vmNetworkMap); SecurityGroup createSecurityGroupForVnfAppliance(DataCenter zone, VirtualMachineTemplate template, Account owner, DeployVnfApplianceCmd cmd); void createIsolatedNetworkRulesForVnfAppliance(DataCenter zone, VirtualMachineTemplate template, Account owner, UserVm vm, DeployVnfApplianceCmd cmd) throws InsufficientAddressCapacityException, ResourceAllocationException, ResourceUnavailableException; + } diff --git a/api/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateUtils.java b/api/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateUtils.java index e997a50cec0..16ff2abb564 100644 --- a/api/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateUtils.java +++ b/api/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateUtils.java @@ -16,6 +16,7 @@ // under the License. package org.apache.cloudstack.storage.template; +import com.cloud.agent.api.to.deployasis.OVFNetworkTO; import com.cloud.exception.InvalidParameterValueException; import com.cloud.network.VNF; import com.cloud.storage.Storage; @@ -124,6 +125,9 @@ public class VnfTemplateUtils { public static void validateApiCommandParams(BaseCmd cmd, VirtualMachineTemplate template) { if (cmd instanceof RegisterVnfTemplateCmd) { RegisterVnfTemplateCmd registerCmd = (RegisterVnfTemplateCmd) cmd; + if (registerCmd.isDeployAsIs() && CollectionUtils.isNotEmpty(registerCmd.getVnfNics())) { + throw new InvalidParameterValueException("VNF nics cannot be specified when register a deploy-as-is Template. Please wait until Template settings are read from OVA."); + } validateApiCommandParams(registerCmd.getVnfDetails(), registerCmd.getVnfNics(), registerCmd.getTemplateType()); } else if (cmd instanceof UpdateVnfTemplateCmd) { UpdateVnfTemplateCmd updateCmd = (UpdateVnfTemplateCmd) cmd; @@ -149,4 +153,18 @@ public class VnfTemplateUtils { } } } + + public static void validateDeployAsIsTemplateVnfNics(List ovfNetworks, List vnfNics) { + if (CollectionUtils.isEmpty(vnfNics)) { + return; + } + if (CollectionUtils.isEmpty(ovfNetworks)) { + throw new InvalidParameterValueException("The list of networks read from OVA is empty. Please wait until the template is fully downloaded and processed."); + } + for (VNF.VnfNic vnfNic : vnfNics) { + if (vnfNic.getDeviceId() < ovfNetworks.size() && !vnfNic.isRequired()) { + throw new InvalidParameterValueException(String.format("The VNF nic [device ID: %s ] is required as it is defined in the OVA template.", vnfNic.getDeviceId())); + } + } + } } diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmdTest.java index a7c41b9271b..235acb15eea 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmdTest.java @@ -46,7 +46,7 @@ public class ImportVolumeCmdTest { Long projectId = 5L; long accountId = 6L; - Mockito.when(accountService.finalyzeAccountId(accountName, domainId, projectId, true)).thenReturn(accountId); + Mockito.when(accountService.finalizeAccountId(accountName, domainId, projectId, true)).thenReturn(accountId); ImportVolumeCmd cmd = new ImportVolumeCmd(); ReflectionTestUtils.setField(cmd, "path", path); diff --git a/api/src/test/java/org/apache/cloudstack/api/command/user/userdata/RegisterUserDataCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/user/userdata/RegisterUserDataCmdTest.java index e9605526f86..8fac32d8f92 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/user/userdata/RegisterUserDataCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/user/userdata/RegisterUserDataCmdTest.java @@ -97,7 +97,7 @@ public class RegisterUserDataCmdTest { ReflectionTestUtils.setField(cmd, "name", "testUserdataName"); ReflectionTestUtils.setField(cmd, "userData", "testUserdata"); - when(_accountService.finalyzeAccountId(ACCOUNT_NAME, DOMAIN_ID, PROJECT_ID, true)).thenReturn(200L); + when(_accountService.finalizeAccountId(ACCOUNT_NAME, DOMAIN_ID, PROJECT_ID, true)).thenReturn(200L); Assert.assertEquals("testUserdataName", cmd.getName()); Assert.assertEquals("testUserdata", cmd.getUserData()); diff --git a/client/pom.xml b/client/pom.xml index d8fa433d5be..b8dffe65d4f 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -666,16 +666,22 @@ - ru.concerteza.buildnumber - maven-jgit-buildnumber-plugin - 1.2.6 + org.codehaus.mojo + buildnumber-maven-plugin + 3.2.0 git-buildnumber - extract-buildnumber + create prepare-package + + false + false + true + unknown + @@ -688,11 +694,11 @@ org.apache.cloudstack.ServerDaemon - ${git.branch} - ${git.tag} - ${git.revision} - ${git.revision} - ${git.branch} + ${scmBranch} + ${project.version} + ${buildNumber} + ${buildNumber} + ${scmBranch} diff --git a/client/src/main/java/org/apache/cloudstack/ACSRequestLog.java b/client/src/main/java/org/apache/cloudstack/ACSRequestLog.java index 123d2761e00..249451120f1 100644 --- a/client/src/main/java/org/apache/cloudstack/ACSRequestLog.java +++ b/client/src/main/java/org/apache/cloudstack/ACSRequestLog.java @@ -18,6 +18,7 @@ // package org.apache.cloudstack; +import com.cloud.api.ApiServlet; import com.cloud.utils.StringUtils; import org.eclipse.jetty.server.NCSARequestLog; import org.eclipse.jetty.server.Request; @@ -25,6 +26,7 @@ import org.eclipse.jetty.server.Response; import org.eclipse.jetty.util.DateCache; import org.eclipse.jetty.util.component.LifeCycle; +import java.net.InetAddress; import java.util.Locale; import java.util.TimeZone; @@ -51,9 +53,8 @@ public class ACSRequestLog extends NCSARequestLog { StringBuilder sb = buffers.get(); sb.setLength(0); - sb.append(request.getHttpChannel().getEndPoint() - .getRemoteAddress().getAddress() - .getHostAddress()) + InetAddress remoteAddress = ApiServlet.getClientAddress(request); + sb.append(remoteAddress.getHostAddress()) .append(" - - [") .append(dateCache.format(request.getTimeStamp())) .append("] \"") diff --git a/client/src/main/java/org/apache/cloudstack/ServerDaemon.java b/client/src/main/java/org/apache/cloudstack/ServerDaemon.java index 196695e1fc6..06477fff898 100644 --- a/client/src/main/java/org/apache/cloudstack/ServerDaemon.java +++ b/client/src/main/java/org/apache/cloudstack/ServerDaemon.java @@ -24,15 +24,12 @@ import java.io.IOException; import java.io.InputStream; import java.lang.management.ManagementFactory; import java.net.URL; -import java.util.Arrays; import java.util.Properties; -import com.cloud.api.ApiServer; import org.apache.commons.daemon.Daemon; import org.apache.commons.daemon.DaemonContext; import org.apache.commons.lang3.StringUtils; import org.eclipse.jetty.jmx.MBeanContainer; -import org.eclipse.jetty.server.ForwardedRequestCustomizer; import org.eclipse.jetty.server.HttpConfiguration; import org.eclipse.jetty.server.HttpConnectionFactory; import org.eclipse.jetty.server.RequestLog; @@ -74,11 +71,6 @@ public class ServerDaemon implements Daemon { private static final String BIND_INTERFACE = "bind.interface"; private static final String CONTEXT_PATH = "context.path"; private static final String SESSION_TIMEOUT = "session.timeout"; - private static final String HTTP_ENABLE = "http.enable"; - private static final String HTTP_PORT = "http.port"; - private static final String HTTPS_ENABLE = "https.enable"; - private static final String HTTPS_PORT = "https.port"; - private static final String KEYSTORE_FILE = "https.keystore"; private static final String KEYSTORE_PASSWORD = "https.keystore.password"; private static final String WEBAPP_DIR = "webapp.dir"; private static final String ACCESS_LOG = "access.log"; @@ -140,11 +132,11 @@ public class ServerDaemon implements Daemon { } setBindInterface(properties.getProperty(BIND_INTERFACE, null)); setContextPath(properties.getProperty(CONTEXT_PATH, "/client")); - setHttpEnable(Boolean.valueOf(properties.getProperty(HTTP_ENABLE, "true"))); - setHttpPort(Integer.valueOf(properties.getProperty(HTTP_PORT, "8080"))); - setHttpsEnable(Boolean.valueOf(properties.getProperty(HTTPS_ENABLE, "false"))); - setHttpsPort(Integer.valueOf(properties.getProperty(HTTPS_PORT, "8443"))); - setKeystoreFile(properties.getProperty(KEYSTORE_FILE)); + setHttpEnable(Boolean.valueOf(properties.getProperty(ServerProperties.HTTP_ENABLE, "true"))); + setHttpPort(Integer.valueOf(properties.getProperty(ServerProperties.HTTP_PORT, "8080"))); + setHttpsEnable(Boolean.valueOf(properties.getProperty(ServerProperties.HTTPS_ENABLE, "false"))); + setHttpsPort(Integer.valueOf(properties.getProperty(ServerProperties.HTTPS_PORT, "8443"))); + setKeystoreFile(properties.getProperty(ServerProperties.KEYSTORE_FILE)); setKeystorePassword(properties.getProperty(KEYSTORE_PASSWORD)); setWebAppLocation(properties.getProperty(WEBAPP_DIR)); setAccessLogFile(properties.getProperty(ACCESS_LOG, "access.log")); @@ -193,7 +185,6 @@ public class ServerDaemon implements Daemon { httpConfig.setResponseHeaderSize(8192); httpConfig.setSendServerVersion(false); httpConfig.setSendDateHeader(false); - addForwardingCustomiser(httpConfig); // HTTP Connector createHttpConnector(httpConfig); @@ -216,21 +207,6 @@ public class ServerDaemon implements Daemon { server.join(); } - /** - * Adds a ForwardedRequestCustomizer to the HTTP configuration to handle forwarded headers. - * The header used for forwarding is determined by the ApiServer.listOfForwardHeaders property. - * Only non empty headers are considered and only the first of the comma-separated list is used. - * @param httpConfig the HTTP configuration to which the customizer will be added - */ - private static void addForwardingCustomiser(HttpConfiguration httpConfig) { - ForwardedRequestCustomizer customiser = new ForwardedRequestCustomizer(); - String header = Arrays.stream(ApiServer.listOfForwardHeaders.value().split(",")).findFirst().orElse(null); - if (com.cloud.utils.StringUtils.isNotEmpty(header)) { - customiser.setForwardedForHeader(header); - } - httpConfig.addCustomizer(customiser); - } - @Override public void stop() throws Exception { server.stop(); diff --git a/core/src/main/java/org/apache/cloudstack/storage/command/TemplateOrVolumePostUploadCommand.java b/core/src/main/java/org/apache/cloudstack/storage/command/TemplateOrVolumePostUploadCommand.java index 3ac83031eaf..253a2607a72 100644 --- a/core/src/main/java/org/apache/cloudstack/storage/command/TemplateOrVolumePostUploadCommand.java +++ b/core/src/main/java/org/apache/cloudstack/storage/command/TemplateOrVolumePostUploadCommand.java @@ -57,8 +57,10 @@ public class TemplateOrVolumePostUploadCommand { private String nfsVersion; - public TemplateOrVolumePostUploadCommand(long entityId, String entityUUID, String absolutePath, String checksum, String type, String name, String imageFormat, String dataTo, - String dataToRole) { + private long zoneId; + + public TemplateOrVolumePostUploadCommand(long entityId, String entityUUID, String absolutePath, String checksum, + String type, String name, String imageFormat, String dataTo, String dataToRole, long zoneId) { this.entityId = entityId; this.entityUUID = entityUUID; this.absolutePath = absolutePath; @@ -68,9 +70,7 @@ public class TemplateOrVolumePostUploadCommand { this.imageFormat = imageFormat; this.dataTo = dataTo; this.dataToRole = dataToRole; - } - - public TemplateOrVolumePostUploadCommand() { + this.zoneId = zoneId; } public String getRemoteEndPoint() { @@ -216,4 +216,8 @@ public class TemplateOrVolumePostUploadCommand { public long getProcessTimeout() { return processTimeout; } + + public long getZoneId() { + return zoneId; + } } diff --git a/core/src/main/java/org/apache/cloudstack/storage/to/SnapshotObjectTO.java b/core/src/main/java/org/apache/cloudstack/storage/to/SnapshotObjectTO.java index 0c3bb99e75c..7643f80bbaa 100644 --- a/core/src/main/java/org/apache/cloudstack/storage/to/SnapshotObjectTO.java +++ b/core/src/main/java/org/apache/cloudstack/storage/to/SnapshotObjectTO.java @@ -48,9 +48,7 @@ public class SnapshotObjectTO extends DownloadableObjectTO implements DataTO { private Long physicalSize = (long) 0; private long accountId; - public SnapshotObjectTO() { - } @Override diff --git a/debian/control b/debian/control index 78842e38ed2..2b8ce929c63 100644 --- a/debian/control +++ b/debian/control @@ -4,7 +4,7 @@ Priority: extra Maintainer: The Apache CloudStack Team Build-Depends: debhelper (>= 9), openjdk-17-jdk | java17-sdk | java17-jdk | zulu-17 | openjdk-11-jdk | java11-sdk | java11-jdk | zulu-11, genisoimage, python-mysql.connector | python3-mysql.connector | mysql-connector-python-py3, maven (>= 3) | maven3, - python (>= 2.7) | python2 (>= 2.7), python3 (>= 3), python-setuptools, python3-setuptools, + python3 (>= 3), python3-setuptools, nodejs (>= 12), lsb-release, dh-systemd | debhelper (>= 13) Standards-Version: 3.8.1 Homepage: http://www.cloudstack.org/ diff --git a/developer/pom.xml b/developer/pom.xml index e2fd782fd25..0a0979ee037 100644 --- a/developer/pom.xml +++ b/developer/pom.xml @@ -66,7 +66,7 @@ org.codehaus.mojo properties-maven-plugin - 1.0-alpha-2 + 1.2.1 initialize diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/StorageOrchestrationService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/StorageOrchestrationService.java index 8be2015bfef..4af0c806060 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/StorageOrchestrationService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/StorageOrchestrationService.java @@ -22,7 +22,6 @@ import java.util.concurrent.Future; import org.apache.cloudstack.api.response.MigrationResponse; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; -import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService.TemplateApiResult; import org.apache.cloudstack.storage.ImageStoreService.MigrationPolicy; @@ -31,5 +30,5 @@ public interface StorageOrchestrationService { MigrationResponse migrateResources(Long srcImgStoreId, Long destImgStoreId, List templateIdList, List snapshotIdList); - Future orchestrateTemplateCopyToImageStore(TemplateInfo source, DataStore destStore); + Future orchestrateTemplateCopyFromSecondaryStores(long templateId, DataStore destStore); } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java index a8861d5acc6..269eb4f1c21 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java @@ -80,4 +80,6 @@ public interface TemplateService { List getTemplateDatadisksOnImageStore(TemplateInfo templateInfo, String configurationId); AsyncCallFuture copyTemplateToImageStore(DataObject source, DataStore destStore); -} + + void handleTemplateCopyFromSecondaryStores(long templateId, DataStore destStore); + } diff --git a/engine/components-api/src/main/java/com/cloud/alert/AlertManager.java b/engine/components-api/src/main/java/com/cloud/alert/AlertManager.java index 3d4e6579f7c..46993b066a4 100644 --- a/engine/components-api/src/main/java/com/cloud/alert/AlertManager.java +++ b/engine/components-api/src/main/java/com/cloud/alert/AlertManager.java @@ -49,10 +49,13 @@ public interface AlertManager extends Manager, AlertService { "Percentage (as a value between 0 and 1) of guest network IPv6 subnet utilization above which alerts will be sent.", true); + ConfigKey AllowedRepetitiveAlertTypes = new ConfigKey<>(ConfigKey.CATEGORY_ALERT, String.class, + "alert.allowed.repetitive.types", "", + "Comma-separated list of alert types (by name) that can be sent multiple times", true); + void clearAlert(AlertType alertType, long dataCenterId, long podId); void recalculateCapacity(); void sendAlert(AlertType alertType, long dataCenterId, Long podId, String subject, String body); - } diff --git a/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java b/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java index 8550dfdd906..f4651237f32 100755 --- a/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java +++ b/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java @@ -57,8 +57,8 @@ public interface ResourceManager extends ResourceService, Configurable { ConfigKey KvmSshToAgentEnabled = new ConfigKey<>("Advanced", Boolean.class, "kvm.ssh.to.agent","true", - "Number of retries when preparing a host into Maintenance Mode is faulty before failing", - false); + "True if the management server will restart the agent service via SSH into the KVM hosts after or during maintenance operations", + true); ConfigKey HOST_MAINTENANCE_LOCAL_STRATEGY = new ConfigKey<>(String.class, "host.maintenance.local.storage.strategy", "Advanced","Error", @@ -79,6 +79,11 @@ public interface ResourceManager extends ResourceService, Configurable { ConfigKey.Kind.Select, "," + CPU.CPUArch.getTypesAsCSV()); + ConfigKey SystemVMDefaultHypervisor = new ConfigKey(String.class, + "system.vm.default.hypervisor", "Advanced", "Any", "Hypervisor type used to create System VMs. Valid values are: XenServer, KVM, VMware, Hyperv, VirtualBox, " + + "Parralels, BareMetal, Ovm, LXC, Any", true, ConfigKey.Scope.Global, null, null, null, null, null, ConfigKey.Kind.Select, "XenServer, KVM, VMware, Hyperv, " + + "VirtualBox, Parralels, BareMetal, Ovm, LXC, Any"); + /** * Register a listener for different types of resource life cycle events. * There can only be one type of listener per type of host. diff --git a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java index 99504d4bc45..3c62738f9ed 100644 --- a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java +++ b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java @@ -228,8 +228,9 @@ public interface StorageManager extends StorageService { ConfigKey.Scope.Global, null); - ConfigKey COPY_PUBLIC_TEMPLATES_FROM_OTHER_STORAGES = new ConfigKey<>(Boolean.class, "copy.public.templates.from.other.storages", - "Storage", "true", "Allow SSVMs to try copying public templates from one secondary storage to another instead of downloading them from the source.", + ConfigKey COPY_TEMPLATES_FROM_OTHER_SECONDARY_STORAGES = new ConfigKey<>(Boolean.class, "copy.templates.from.other.secondary.storages", + "Storage", "true", "When enabled, this feature allows templates to be copied from existing Secondary Storage servers (within the same zone or across zones) " + + "while adding a new Secondary Storage. If the copy operation fails, the system falls back to downloading the template from the source URL.", true, ConfigKey.Scope.Zone, null); /** diff --git a/engine/components-api/src/main/java/com/cloud/template/TemplateManager.java b/engine/components-api/src/main/java/com/cloud/template/TemplateManager.java index 168b4870909..f1891c774ed 100644 --- a/engine/components-api/src/main/java/com/cloud/template/TemplateManager.java +++ b/engine/components-api/src/main/java/com/cloud/template/TemplateManager.java @@ -153,6 +153,8 @@ public interface TemplateManager { TemplateType validateTemplateType(BaseCmd cmd, boolean isAdmin, boolean isCrossZones, Hypervisor.HypervisorType hypervisorType); + DataStore verifyHeuristicRulesForZone(VMTemplateVO template, Long zoneId); + List getTemplateDisksOnImageStore(VirtualMachineTemplate template, DataStoreRole role, String configurationId); static Boolean getValidateUrlIsResolvableBeforeRegisteringTemplateValue() { diff --git a/engine/components-api/src/main/java/com/cloud/vm/snapshot/VMSnapshotManager.java b/engine/components-api/src/main/java/com/cloud/vm/snapshot/VMSnapshotManager.java index 6478469f190..6831552b83d 100644 --- a/engine/components-api/src/main/java/com/cloud/vm/snapshot/VMSnapshotManager.java +++ b/engine/components-api/src/main/java/com/cloud/vm/snapshot/VMSnapshotManager.java @@ -31,7 +31,7 @@ public interface VMSnapshotManager extends VMSnapshotService, Manager { static final ConfigKey VMSnapshotExpireInterval = new ConfigKey("Advanced", Integer.class, "vmsnapshot.expire.interval", "-1", "VM Snapshot expire interval in hours", true, ConfigKey.Scope.Account); - ConfigKey VMSnapshotMax = new ConfigKey("Advanced", Integer.class, "vmsnapshot.max", "10", "Maximum vm snapshots for a single vm", true, ConfigKey.Scope.Global); + ConfigKey VMSnapshotMax = new ConfigKey("Advanced", Integer.class, "vmsnapshot.max", "10", "Maximum VM snapshots for a single VM", true, ConfigKey.Scope.Account); /** * Delete all VM snapshots belonging to one VM diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 86f45630611..e8796fb0252 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -935,7 +935,11 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac throw new CloudRuntimeException(String.format("Unable to start a VM [%s] due to [%s].", vmUuid, e.getMessage()), e).add(VirtualMachine.class, vmUuid); } catch (final ResourceUnavailableException e) { if (e.getScope() != null && e.getScope().equals(VirtualRouter.class)){ - throw new CloudRuntimeException("Network is unavailable. Please contact administrator", e).add(VirtualMachine.class, vmUuid); + Account callingAccount = CallContext.current().getCallingAccount(); + String errorSuffix = (callingAccount != null && callingAccount.getType() == Account.Type.ADMIN) ? + String.format("Failure: %s", e.getMessage()) : + "Please contact administrator."; + throw new CloudRuntimeException(String.format("The Network for VM %s is unavailable. %s", vmUuid, errorSuffix), e).add(VirtualMachine.class, vmUuid); } throw new CloudRuntimeException(String.format("Unable to start a VM [%s] due to [%s].", vmUuid, e.getMessage()), e).add(VirtualMachine.class, vmUuid); } diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachinePowerStateSyncImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachinePowerStateSyncImpl.java index 7a1a39ec0f0..475ed0f37bd 100644 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachinePowerStateSyncImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachinePowerStateSyncImpl.java @@ -89,7 +89,7 @@ public class VirtualMachinePowerStateSyncImpl implements VirtualMachinePowerStat return; } for (Long vmId : vmIds) { - if (!notUpdated.containsKey(vmId)) { + if (MapUtils.isEmpty(notUpdated) || !notUpdated.containsKey(vmId)) { logger.debug("VM state report is updated. {}, {}, power state: {}", () -> hostCache.get(hostId), () -> vmCache.get(vmId), () -> instancePowerStates.get(vmId)); _messageBus.publish(null, VirtualMachineManager.Topics.VM_POWER_STATE, @@ -158,8 +158,8 @@ public class VirtualMachinePowerStateSyncImpl implements VirtualMachinePowerStat // an update might have occurred that we should not override in case of out of band migration instancePowerStates.put(instance.getId(), VirtualMachine.PowerState.PowerReportMissing); } else { - logger.debug("vm id: {} - time since last state update({} ms) has not passed graceful period yet", - instance.getId(), milliSecondsSinceLastStateUpdate); + logger.debug("vm id: {} - time since last state update({} ms) has not passed graceful period ({} ms) yet", + instance.getId(), milliSecondsSinceLastStateUpdate, milliSecondsGracefulPeriod); } } updateAndPublishVmPowerStates(hostId, instancePowerStates, startTime); diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java index d1ab423bc6b..31144296f60 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java @@ -3586,8 +3586,9 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra final HashMap stillFree = new HashMap<>(); final List networkIds = _networksDao.findNetworksToGarbageCollect(); - final int netGcWait = NumbersUtil.parseInt(_configDao.getValue(NetworkGcWait.key()), 60); - logger.info("NetworkGarbageCollector uses '{}' seconds for GC interval.", netGcWait); + final int netGcWait = NetworkGcWait.value(); + final int netGcInterval = NetworkGcInterval.value(); + logger.info("NetworkGarbageCollector uses '{}' seconds for GC wait and '{}' seconds for GC interval.", netGcWait, netGcInterval); for (final Long networkId : networkIds) { if (!_networkModel.isNetworkReadyForGc(networkId)) { @@ -4908,10 +4909,10 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra return NetworkOrchestrationService.class.getSimpleName(); } - public static final ConfigKey NetworkGcWait = new ConfigKey<>(Integer.class, "network.gc.wait", "Advanced", "600", - "Time (in seconds) to wait before shutting down a network that's not in used", false, Scope.Global, null); - public static final ConfigKey NetworkGcInterval = new ConfigKey<>(Integer.class, "network.gc.interval", "Advanced", "600", - "Seconds to wait before checking for networks to shutdown", true, Scope.Global, null); + public static final ConfigKey NetworkGcWait = new ConfigKey(Integer.class, "network.gc.wait", "Advanced", "600", + "Time (in seconds) to wait before shutting down a network that's not in used", true, Scope.Global, null); + public static final ConfigKey NetworkGcInterval = new ConfigKey(Integer.class, "network.gc.interval", "Advanced", "600", + "Seconds to wait before checking for networks to shutdown", false, Scope.Global, null); @Override public ConfigKey[] getConfigKeys() { diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java index 37a1f8dc196..933b4e0c5ce 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java @@ -36,6 +36,9 @@ import java.util.stream.Collectors; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.template.TemplateManager; import org.apache.cloudstack.api.response.MigrationResponse; import org.apache.cloudstack.engine.orchestration.service.StorageOrchestrationService; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; @@ -45,6 +48,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.SecondaryStorageServic import org.apache.cloudstack.engine.subsystem.api.storage.SecondaryStorageService.DataObjectResult; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService.TemplateApiResult; @@ -103,6 +107,15 @@ public class StorageOrchestrator extends ManagerBase implements StorageOrchestra VolumeDataStoreDao volumeDataStoreDao; @Inject DataMigrationUtility migrationHelper; + @Inject + TemplateManager templateManager; + @Inject + VMTemplateDao templateDao; + @Inject + TemplateDataFactory templateDataFactory; + @Inject + DataCenterDao dcDao; + ConfigKey ImageStoreImbalanceThreshold = new ConfigKey<>("Advanced", Double.class, "image.store.imbalance.threshold", @@ -304,8 +317,9 @@ public class StorageOrchestrator extends ManagerBase implements StorageOrchestra } @Override - public Future orchestrateTemplateCopyToImageStore(TemplateInfo source, DataStore destStore) { - return submit(destStore.getScope().getScopeId(), new CopyTemplateTask(source, destStore)); + public Future orchestrateTemplateCopyFromSecondaryStores(long srcTemplateId, DataStore destStore) { + Long dstZoneId = destStore.getScope().getScopeId(); + return submit(dstZoneId, new CopyTemplateFromSecondaryStorageTask(srcTemplateId, destStore)); } protected Pair migrateCompleted(Long destDatastoreId, DataStore srcDatastore, List files, MigrationPolicy migrationPolicy, int skipped) { @@ -624,13 +638,13 @@ public class StorageOrchestrator extends ManagerBase implements StorageOrchestra } } - private class CopyTemplateTask implements Callable { - private TemplateInfo sourceTmpl; - private DataStore destStore; - private String logid; + private class CopyTemplateFromSecondaryStorageTask implements Callable { + private final long srcTemplateId; + private final DataStore destStore; + private final String logid; - public CopyTemplateTask(TemplateInfo sourceTmpl, DataStore destStore) { - this.sourceTmpl = sourceTmpl; + CopyTemplateFromSecondaryStorageTask(long srcTemplateId, DataStore destStore) { + this.srcTemplateId = srcTemplateId; this.destStore = destStore; this.logid = ThreadContext.get(LOGCONTEXTID); } @@ -639,17 +653,16 @@ public class StorageOrchestrator extends ManagerBase implements StorageOrchestra public TemplateApiResult call() { ThreadContext.put(LOGCONTEXTID, logid); TemplateApiResult result; - AsyncCallFuture future = templateService.copyTemplateToImageStore(sourceTmpl, destStore); + long destZoneId = destStore.getScope().getScopeId(); + TemplateInfo sourceTmpl = templateDataFactory.getTemplate(srcTemplateId, DataStoreRole.Image); try { - result = future.get(); - } catch (ExecutionException | InterruptedException e) { - logger.warn("Exception while copying template [{}] from image store [{}] to image store [{}]: {}", - sourceTmpl.getUniqueName(), sourceTmpl.getDataStore().getName(), destStore.getName(), e.toString()); + templateService.handleTemplateCopyFromSecondaryStores(srcTemplateId, destStore); result = new TemplateApiResult(sourceTmpl); - result.setResult(e.getMessage()); + } finally { + tryCleaningUpExecutor(destZoneId); + ThreadContext.clearAll(); } - tryCleaningUpExecutor(destStore.getScope().getScopeId()); - ThreadContext.clearAll(); + return result; } } diff --git a/engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java b/engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java index 8066b89b4b9..4e8b6204f72 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java @@ -598,7 +598,7 @@ public class NetworkDaoImpl extends GenericDaoBaseimplements Ne public List listByPhysicalNetworkTrafficType(final long physicalNetworkId, final TrafficType trafficType) { final SearchCriteria sc = AllFieldsSearch.create(); sc.setParameters("trafficType", trafficType); - sc.setParameters("physicalNetwork", physicalNetworkId); + sc.setParameters("physicalNetworkId", physicalNetworkId); return listBy(sc); } diff --git a/engine/schema/src/main/java/com/cloud/network/dao/PhysicalNetworkTrafficTypeDaoImpl.java b/engine/schema/src/main/java/com/cloud/network/dao/PhysicalNetworkTrafficTypeDaoImpl.java index 09d9f1d7fbf..fdd827ffeee 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/PhysicalNetworkTrafficTypeDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/PhysicalNetworkTrafficTypeDaoImpl.java @@ -137,7 +137,9 @@ public class PhysicalNetworkTrafficTypeDaoImpl extends GenericDaoBase tag = customSearch(sc, null); return tag.size() == 0 ? null : tag.get(0); diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java index 44c0be1cb59..a03b94faa79 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java @@ -48,7 +48,7 @@ public interface VolumeDao extends GenericDao, StateDao findIncludingRemovedByInstanceAndType(long id, Volume.Type vType); - List findByInstanceIdAndPoolId(long instanceId, long poolId); + List findNonDestroyedVolumesByInstanceIdAndPoolId(long instanceId, long poolId); List findByInstanceIdDestroyed(long vmId); @@ -70,11 +70,11 @@ public interface VolumeDao extends GenericDao, StateDao findCreatedByInstance(long id); - List findByPoolId(long poolId); + List findNonDestroyedVolumesByPoolId(long poolId); VolumeVO findByPoolIdName(long poolId, String name); - List findByPoolId(long poolId, Volume.Type volumeType); + List findNonDestroyedVolumesByPoolId(long poolId, Volume.Type volumeType); List findByPoolIdAndState(long poolid, Volume.State state); diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java index 2e263caeea3..286d4ff7542 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java @@ -136,7 +136,7 @@ public class VolumeDaoImpl extends GenericDaoBase implements Vol } @Override - public List findByPoolId(long poolId) { + public List findNonDestroyedVolumesByPoolId(long poolId) { SearchCriteria sc = AllFieldsSearch.create(); sc.setParameters("poolId", poolId); sc.setParameters("notDestroyed", Volume.State.Destroy, Volume.State.Expunged); @@ -145,7 +145,7 @@ public class VolumeDaoImpl extends GenericDaoBase implements Vol } @Override - public List findByInstanceIdAndPoolId(long instanceId, long poolId) { + public List findNonDestroyedVolumesByInstanceIdAndPoolId(long instanceId, long poolId) { SearchCriteria sc = AllFieldsSearch.create(); sc.setParameters("instanceId", instanceId); sc.setParameters("poolId", poolId); @@ -162,7 +162,7 @@ public class VolumeDaoImpl extends GenericDaoBase implements Vol } @Override - public List findByPoolId(long poolId, Volume.Type volumeType) { + public List findNonDestroyedVolumesByPoolId(long poolId, Volume.Type volumeType) { SearchCriteria sc = AllFieldsSearch.create(); sc.setParameters("poolId", poolId); sc.setParameters("notDestroyed", Volume.State.Destroy, Volume.State.Expunged); diff --git a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java index 79a4bd6d6d8..292bafefbb6 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java @@ -606,7 +606,7 @@ public class SystemVmTemplateRegistration { template.setBits(64); template.setAccountId(Account.ACCOUNT_ID_SYSTEM); template.setUrl(details.getUrl()); - template.setChecksum(details.getChecksum()); + template.setChecksum(DigestHelper.prependAlgorithm(details.getChecksum())); template.setEnablePassword(false); template.setDisplayText(details.getName()); template.setFormat(details.getFormat()); @@ -1079,7 +1079,7 @@ public class SystemVmTemplateRegistration { protected void updateTemplateUrlChecksumAndGuestOsId(VMTemplateVO templateVO, MetadataTemplateDetails templateDetails) { templateVO.setUrl(templateDetails.getUrl()); - templateVO.setChecksum(templateDetails.getChecksum()); + templateVO.setChecksum(DigestHelper.prependAlgorithm(templateDetails.getChecksum())); GuestOSVO guestOS = guestOSDao.findOneByDisplayName(templateDetails.getGuestOs()); if (guestOS != null) { templateVO.setGuestOSId(guestOS.getId()); diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42020to42030.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42020to42030.java new file mode 100644 index 00000000000..68100e16401 --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42020to42030.java @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.upgrade.dao; + +import java.io.InputStream; +import java.sql.Connection; + +import com.cloud.utils.exception.CloudRuntimeException; + +public class Upgrade42020to42030 extends DbUpgradeAbstractImpl implements DbUpgrade, DbUpgradeSystemVmTemplate { + + @Override + public String[] getUpgradableVersionRange() { + return new String[]{"4.20.2.0", "4.20.3.0"}; + } + + @Override + public String getUpgradedVersion() { + return "4.20.3.0"; + } + + @Override + public boolean supportsRollingUpgrade() { + return false; + } + + @Override + public InputStream[] getPrepareScripts() { + final String scriptFile = "META-INF/db/schema-42020to42030.sql"; + final InputStream script = Thread.currentThread().getContextClassLoader().getResourceAsStream(scriptFile); + if (script == null) { + throw new CloudRuntimeException("Unable to find " + scriptFile); + } + + return new InputStream[] {script}; + } + + @Override + public void performDataMigration(Connection conn) { + } + + @Override + public InputStream[] getCleanupScripts() { + return null; + } + + @Override + public void updateSystemVmTemplates(Connection conn) { + } +} diff --git a/engine/schema/src/main/java/com/cloud/usage/dao/UsageDao.java b/engine/schema/src/main/java/com/cloud/usage/dao/UsageDao.java index ea490e60f9e..b9d8d9c0536 100644 --- a/engine/schema/src/main/java/com/cloud/usage/dao/UsageDao.java +++ b/engine/schema/src/main/java/com/cloud/usage/dao/UsageDao.java @@ -62,7 +62,7 @@ public interface UsageDao extends GenericDao { void saveUsageRecords(List usageRecords); - void removeOldUsageRecords(int days); + void expungeAllOlderThan(int days, long limitPerQuery); UsageVO persistUsage(final UsageVO usage); diff --git a/engine/schema/src/main/java/com/cloud/usage/dao/UsageDaoImpl.java b/engine/schema/src/main/java/com/cloud/usage/dao/UsageDaoImpl.java index 34e7843bfc4..2d99c78fad1 100644 --- a/engine/schema/src/main/java/com/cloud/usage/dao/UsageDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/usage/dao/UsageDaoImpl.java @@ -26,15 +26,16 @@ import com.cloud.utils.Pair; import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.QueryBuilder; +import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.Transaction; import com.cloud.utils.db.TransactionCallback; -import com.cloud.utils.db.TransactionCallbackNoReturn; import com.cloud.utils.db.TransactionLegacy; import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.acl.RoleType; +import org.apache.commons.lang3.time.DateUtils; import org.springframework.stereotype.Component; import java.sql.PreparedStatement; @@ -51,7 +52,6 @@ import java.util.TimeZone; public class UsageDaoImpl extends GenericDaoBase implements UsageDao { private static final String DELETE_ALL = "DELETE FROM cloud_usage"; private static final String DELETE_ALL_BY_ACCOUNTID = "DELETE FROM cloud_usage WHERE account_id = ?"; - private static final String DELETE_ALL_BY_INTERVAL = "DELETE FROM cloud_usage WHERE end_date < DATE_SUB(CURRENT_DATE(), INTERVAL ? DAY)"; private static final String INSERT_ACCOUNT = "INSERT INTO cloud_usage.account (id, account_name, uuid, type, role_id, domain_id, removed, cleanup_needed) VALUES (?,?,?,?,?,?,?,?)"; private static final String INSERT_USER_STATS = "INSERT INTO cloud_usage.user_statistics (id, data_center_id, account_id, public_ip_address, device_id, device_type, network_id, net_bytes_received," + " net_bytes_sent, current_bytes_received, current_bytes_sent, agg_bytes_received, agg_bytes_sent) VALUES (?,?,?,?,?,?,?,?,?,?, ?, ?, ?)"; @@ -88,8 +88,12 @@ public class UsageDaoImpl extends GenericDaoBase implements Usage private static final String UPDATE_BUCKET_STATS = "UPDATE cloud_usage.bucket_statistics SET size=? WHERE id=?"; + protected SearchBuilder endDateLessThanSearch; public UsageDaoImpl() { + endDateLessThanSearch = createSearchBuilder(); + endDateLessThanSearch.and("endDate", endDateLessThanSearch.entity().getEndDate(), SearchCriteria.Op.LT); + endDateLessThanSearch.done(); } @Override @@ -539,21 +543,20 @@ public class UsageDaoImpl extends GenericDaoBase implements Usage } @Override - public void removeOldUsageRecords(int days) { - Transaction.execute(TransactionLegacy.USAGE_DB, new TransactionCallbackNoReturn() { - @Override - public void doInTransactionWithoutResult(TransactionStatus status) { - TransactionLegacy txn = TransactionLegacy.currentTxn(); - PreparedStatement pstmt = null; - try { - pstmt = txn.prepareAutoCloseStatement(DELETE_ALL_BY_INTERVAL); - pstmt.setLong(1, days); - pstmt.executeUpdate(); - } catch (Exception ex) { - logger.error("error removing old cloud_usage records for interval: " + days); - } - } - }); + public void expungeAllOlderThan(int days, long limitPerQuery) { + SearchCriteria sc = endDateLessThanSearch.create(); + + Date limit = DateUtils.addDays(new Date(), -days); + sc.setParameters("endDate", limit); + + TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.USAGE_DB); + try { + logger.debug("Removing all cloud_usage records older than [{}].", limit); + int totalExpunged = batchExpunge(sc, limitPerQuery); + logger.info("Removed a total of [{}] cloud_usage records older than [{}].", totalExpunged, limit); + } finally { + txn.close(); + } } public UsageVO persistUsage(final UsageVO usage) { diff --git a/engine/schema/src/main/java/com/cloud/usage/dao/UsageJobDao.java b/engine/schema/src/main/java/com/cloud/usage/dao/UsageJobDao.java index d4038d4ceeb..b22ce69d94e 100644 --- a/engine/schema/src/main/java/com/cloud/usage/dao/UsageJobDao.java +++ b/engine/schema/src/main/java/com/cloud/usage/dao/UsageJobDao.java @@ -28,6 +28,8 @@ public interface UsageJobDao extends GenericDao { UsageJobVO getLastJob(); + UsageJobVO getNextRecurringJob(); + UsageJobVO getNextImmediateJob(); long getLastJobSuccessDateMillis(); diff --git a/engine/schema/src/main/java/com/cloud/usage/dao/UsageJobDaoImpl.java b/engine/schema/src/main/java/com/cloud/usage/dao/UsageJobDaoImpl.java index 44a7d1a8b72..6f340501cf1 100644 --- a/engine/schema/src/main/java/com/cloud/usage/dao/UsageJobDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/usage/dao/UsageJobDaoImpl.java @@ -156,7 +156,8 @@ public class UsageJobDaoImpl extends GenericDaoBase implements return jobs.get(0); } - private UsageJobVO getNextRecurringJob() { + @Override + public UsageJobVO getNextRecurringJob() { Filter filter = new Filter(UsageJobVO.class, "id", false, Long.valueOf(0), Long.valueOf(1)); SearchCriteria sc = createSearchCriteria(); sc.addAnd("endMillis", SearchCriteria.Op.EQ, Long.valueOf(0)); diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDetailsDao.java b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDetailsDao.java index ea9ac5afba6..4cbdc516ba0 100644 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDetailsDao.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDetailsDao.java @@ -22,4 +22,5 @@ import com.cloud.utils.db.GenericDao; import com.cloud.vm.VMInstanceDetailVO; public interface VMInstanceDetailsDao extends GenericDao, ResourceDetailsDao { + int removeDetailsWithPrefix(long vmId, String prefix); } diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDetailsDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDetailsDaoImpl.java index ca11b005fb2..4c2fdd6f8d4 100644 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDetailsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDetailsDaoImpl.java @@ -17,10 +17,13 @@ package com.cloud.vm.dao; +import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; import org.apache.cloudstack.resourcedetail.ResourceDetailsDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; import com.cloud.vm.VMInstanceDetailVO; @Component @@ -31,4 +34,18 @@ public class VMInstanceDetailsDaoImpl extends ResourceDetailsDaoBase sb = createSearchBuilder(); + sb.and("vmId", sb.entity().getResourceId(), SearchCriteria.Op.EQ); + sb.and("prefix", sb.entity().getName(), SearchCriteria.Op.LIKE); + sb.done(); + SearchCriteria sc = sb.create(); + sc.setParameters("vmId", vmId); + sc.setParameters("prefix", prefix + "%"); + return super.remove(sc); + } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java index e0e7b4a6ba2..6aeee1ad1cc 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java @@ -130,4 +130,18 @@ StateDao snapshotIds, Long batchSize); + + /** + * Returns the total physical size, in bytes, of all snapshots stored on primary + * storage for the specified account that have not yet been backed up to + * secondary storage. + * + *

If no such snapshots are found, this method returns {@code 0}.

+ * + * @param accountId the ID of the account whose snapshots on primary storage + * should be considered + * @return the total physical size in bytes of matching snapshots on primary + * storage, or {@code 0} if none are found + */ + long getSnapshotsPhysicalSizeOnPrimaryStorageByAccountId(long accountId); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java index ac9601389bd..cdf903407c1 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java @@ -67,7 +67,7 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase searchFilteringStoreIdEqStoreRoleEqStateNeqRefCntNeq; protected SearchBuilder searchFilteringStoreIdEqStateEqStoreRoleEqIdEqUpdateCountEqSnapshotIdEqVolumeIdEq; private SearchBuilder stateSearch; - private SearchBuilder idStateNinSearch; + private SearchBuilder idStateNeqSearch; protected SearchBuilder snapshotVOSearch; private SearchBuilder snapshotCreatedSearch; private SearchBuilder dataStoreAndInstallPathSearch; @@ -96,6 +96,15 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase params) throws ConfigurationException { super.configure(name, params); @@ -137,10 +146,10 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase findBySnapshotIdWithNonDestroyedState(long snapshotId) { - SearchCriteria sc = idStateNinSearch.create(); + SearchCriteria sc = idStateNeqSearch.create(); sc.setParameters(SNAPSHOT_ID, snapshotId); sc.setParameters(STATE, State.Destroyed.name()); return listBy(sc); @@ -479,7 +488,7 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase findBySnapshotIdAndNotInDestroyedHiddenState(long snapshotId) { - SearchCriteria sc = idStateNinSearch.create(); + SearchCriteria sc = idStateNeqSearch.create(); sc.setParameters(SNAPSHOT_ID, snapshotId); sc.setParameters(STATE, State.Destroyed.name(), State.Hidden.name()); return listBy(sc); @@ -732,4 +741,23 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase { int expungeJobsForSchedules(List scheduleId, Date dateAfter); int expungeJobsBefore(Date currentTimestamp); + + VMScheduledJobVO findByScheduleAndTimestamp(long scheduleId, Date scheduledTimestamp); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/vm/schedule/dao/VMScheduledJobDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/vm/schedule/dao/VMScheduledJobDaoImpl.java index 50a2b12fd77..2f08a41b92e 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/vm/schedule/dao/VMScheduledJobDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/vm/schedule/dao/VMScheduledJobDaoImpl.java @@ -39,6 +39,8 @@ public class VMScheduledJobDaoImpl extends GenericDaoBase expungeJobForScheduleSearch; + private final SearchBuilder scheduleAndTimestampSearch; + static final String SCHEDULED_TIMESTAMP = "scheduled_timestamp"; static final String VM_SCHEDULE_ID = "vm_schedule_id"; @@ -58,6 +60,11 @@ public class VMScheduledJobDaoImpl extends GenericDaoBase sc = scheduleAndTimestampSearch.create(); + sc.setParameters(VM_SCHEDULE_ID, scheduleId); + sc.setParameters(SCHEDULED_TIMESTAMP, scheduledTimestamp); + return findOneBy(sc); + } } diff --git a/engine/schema/src/main/resources/META-INF/db/procedures/cloud.insert_category_if_not_exists.sql b/engine/schema/src/main/resources/META-INF/db/procedures/cloud.insert_category_if_not_exists.sql new file mode 100644 index 00000000000..a82dc7204c2 --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/procedures/cloud.insert_category_if_not_exists.sql @@ -0,0 +1,27 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Add new OS categories if not present +DROP PROCEDURE IF EXISTS `cloud`.`INSERT_CATEGORY_IF_NOT_EXIST`; +CREATE PROCEDURE `cloud`.`INSERT_CATEGORY_IF_NOT_EXIST`(IN os_name VARCHAR(255)) +BEGIN + IF NOT EXISTS ((SELECT 1 FROM `cloud`.`guest_os_category` WHERE name = os_name)) + THEN + INSERT INTO `cloud`.`guest_os_category` (name, uuid) + VALUES (os_name, UUID()) +; END IF +; END; diff --git a/engine/schema/src/main/resources/META-INF/db/procedures/cloud.insert_extension_custom_action_details_if_not_exists.sql b/engine/schema/src/main/resources/META-INF/db/procedures/cloud.insert_extension_custom_action_details_if_not_exists.sql new file mode 100644 index 00000000000..77b16223626 --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/procedures/cloud.insert_extension_custom_action_details_if_not_exists.sql @@ -0,0 +1,46 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +DROP PROCEDURE IF EXISTS `cloud`.`INSERT_EXTENSION_CUSTOM_ACTION_DETAILS_IF_NOT_EXISTS`; +CREATE PROCEDURE `cloud`.`INSERT_EXTENSION_CUSTOM_ACTION_DETAILS_IF_NOT_EXISTS` ( + IN ext_name VARCHAR(255), + IN action_name VARCHAR(255), + IN param_json TEXT +) +BEGIN + DECLARE action_id BIGINT UNSIGNED +; SELECT `eca`.`id` INTO action_id FROM `cloud`.`extension_custom_action` `eca` + JOIN `cloud`.`extension` `e` ON `e`.`id` = `eca`.`extension_id` + WHERE `eca`.`name` = action_name AND `e`.`name` = ext_name LIMIT 1 +; IF NOT EXISTS ( + SELECT 1 FROM `cloud`.`extension_custom_action_details` + WHERE `extension_custom_action_id` = action_id + AND `name` = 'parameters' + ) THEN + INSERT INTO `cloud`.`extension_custom_action_details` ( + `extension_custom_action_id`, + `name`, + `value`, + `display` + ) VALUES ( + action_id, + 'parameters', + param_json, + 0 + ) +; END IF +;END; diff --git a/engine/schema/src/main/resources/META-INF/db/procedures/cloud.insert_extension_custom_action_if_not_exists.sql b/engine/schema/src/main/resources/META-INF/db/procedures/cloud.insert_extension_custom_action_if_not_exists.sql new file mode 100644 index 00000000000..9dbffa630f8 --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/procedures/cloud.insert_extension_custom_action_if_not_exists.sql @@ -0,0 +1,46 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +DROP PROCEDURE IF EXISTS `cloud`.`INSERT_EXTENSION_CUSTOM_ACTION_IF_NOT_EXISTS`; +CREATE PROCEDURE `cloud`.`INSERT_EXTENSION_CUSTOM_ACTION_IF_NOT_EXISTS`( + IN ext_name VARCHAR(255), + IN action_name VARCHAR(255), + IN action_desc VARCHAR(4096), + IN resource_type VARCHAR(255), + IN allowed_roles INT UNSIGNED, + IN success_msg VARCHAR(4096), + IN error_msg VARCHAR(4096), + IN timeout_seconds INT UNSIGNED +) +BEGIN + DECLARE ext_id BIGINT +; SELECT `id` INTO ext_id FROM `cloud`.`extension` WHERE `name` = ext_name LIMIT 1 +; IF NOT EXISTS ( + SELECT 1 FROM `cloud`.`extension_custom_action` WHERE `name` = action_name AND `extension_id` = ext_id + ) THEN + INSERT INTO `cloud`.`extension_custom_action` ( + `uuid`, `name`, `description`, `extension_id`, `resource_type`, + `allowed_role_types`, `success_message`, `error_message`, + `enabled`, `timeout`, `created`, `removed` + ) + VALUES ( + UUID(), action_name, action_desc, ext_id, resource_type, + allowed_roles, success_msg, error_msg, + 1, timeout_seconds, NOW(), NULL + ) +; END IF +;END; diff --git a/engine/schema/src/main/resources/META-INF/db/procedures/cloud.insert_extension_detail_if_not_exists.sql b/engine/schema/src/main/resources/META-INF/db/procedures/cloud.insert_extension_detail_if_not_exists.sql new file mode 100644 index 00000000000..f9d6c5da951 --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/procedures/cloud.insert_extension_detail_if_not_exists.sql @@ -0,0 +1,39 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +DROP PROCEDURE IF EXISTS `cloud`.`INSERT_EXTENSION_DETAIL_IF_NOT_EXISTS`; +CREATE PROCEDURE `cloud`.`INSERT_EXTENSION_DETAIL_IF_NOT_EXISTS`( + IN ext_name VARCHAR(255), + IN detail_key VARCHAR(255), + IN detail_value TEXT, + IN display TINYINT(1) +) +BEGIN + DECLARE ext_id BIGINT +; SELECT `id` INTO ext_id FROM `cloud`.`extension` WHERE `name` = ext_name LIMIT 1 +; IF NOT EXISTS ( + SELECT 1 FROM `cloud`.`extension_details` + WHERE `extension_id` = ext_id AND `name` = detail_key + ) THEN + INSERT INTO `cloud`.`extension_details` ( + `extension_id`, `name`, `value`, `display` + ) + VALUES ( + ext_id, detail_key, detail_value, display + ) +; END IF +;END; diff --git a/engine/schema/src/main/resources/META-INF/db/procedures/cloud.insert_extension_if_not_exists.sql b/engine/schema/src/main/resources/META-INF/db/procedures/cloud.insert_extension_if_not_exists.sql new file mode 100644 index 00000000000..8d74f9b2a98 --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/procedures/cloud.insert_extension_if_not_exists.sql @@ -0,0 +1,38 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +DROP PROCEDURE IF EXISTS `cloud`.`INSERT_EXTENSION_IF_NOT_EXISTS`; +CREATE PROCEDURE `cloud`.`INSERT_EXTENSION_IF_NOT_EXISTS`( + IN ext_name VARCHAR(255), + IN ext_desc VARCHAR(255), + IN ext_path VARCHAR(255) +) +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM `cloud`.`extension` WHERE `name` = ext_name + ) THEN + INSERT INTO `cloud`.`extension` ( + `uuid`, `name`, `description`, `type`, + `relative_path`, `path_ready`, + `is_user_defined`, `state`, `created`, `removed` + ) + VALUES ( + UUID(), ext_name, ext_desc, 'Orchestrator', + ext_path, 1, 0, 'Enabled', NOW(), NULL + ) +; END IF +;END; diff --git a/engine/schema/src/main/resources/META-INF/db/procedures/cloud.update_category_for_guest_oses.sql b/engine/schema/src/main/resources/META-INF/db/procedures/cloud.update_category_for_guest_oses.sql new file mode 100644 index 00000000000..87f3a85d27e --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/procedures/cloud.update_category_for_guest_oses.sql @@ -0,0 +1,33 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Move existing guest OS to new categories +DROP PROCEDURE IF EXISTS `cloud`.`UPDATE_CATEGORY_FOR_GUEST_OSES`; +CREATE PROCEDURE `cloud`.`UPDATE_CATEGORY_FOR_GUEST_OSES`(IN category_name VARCHAR(255), IN os_name VARCHAR(255)) +BEGIN + DECLARE category_id BIGINT +; SELECT `id` INTO category_id + FROM `cloud`.`guest_os_category` + WHERE `name` = category_name + LIMIT 1 +; IF category_id IS NULL THEN + SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = 'Category not found' +; END IF +; UPDATE `cloud`.`guest_os` + SET `category_id` = category_id + WHERE `display_name` LIKE CONCAT('%', os_name, '%') +; END; diff --git a/engine/schema/src/main/resources/META-INF/db/procedures/cloud.update_new_and_delete_old_category_for_guest_os.sql b/engine/schema/src/main/resources/META-INF/db/procedures/cloud.update_new_and_delete_old_category_for_guest_os.sql new file mode 100644 index 00000000000..42f7aa738cf --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/procedures/cloud.update_new_and_delete_old_category_for_guest_os.sql @@ -0,0 +1,35 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Move existing guest OS whose category will be deleted to Other category +DROP PROCEDURE IF EXISTS `cloud`.`UPDATE_NEW_AND_DELETE_OLD_CATEGORY_FOR_GUEST_OS`; +CREATE PROCEDURE `cloud`.`UPDATE_NEW_AND_DELETE_OLD_CATEGORY_FOR_GUEST_OS`(IN to_category_name VARCHAR(255), IN from_category_name VARCHAR(255)) +BEGIN + DECLARE done INT DEFAULT 0 +; DECLARE to_category_id BIGINT +; SELECT id INTO to_category_id + FROM `cloud`.`guest_os_category` + WHERE `name` = to_category_name + LIMIT 1 +; IF to_category_id IS NULL THEN + SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = 'ToCategory not found' +; END IF +; UPDATE `cloud`.`guest_os` + SET `category_id` = to_category_id + WHERE `category_id` = (SELECT `id` FROM `cloud`.`guest_os_category` WHERE `name` = from_category_name) +; UPDATE `cloud`.`guest_os_category` SET `removed`=now() WHERE `name` = from_category_name +; END; diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42020to42030.sql b/engine/schema/src/main/resources/META-INF/db/schema-42020to42030.sql new file mode 100644 index 00000000000..5eec97278ba --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/schema-42020to42030.sql @@ -0,0 +1,28 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +--; +-- Schema upgrade from 4.20.2.0 to 4.20.3.0 +--; + +ALTER TABLE `cloud`.`template_store_ref` MODIFY COLUMN `download_url` varchar(2048); + +UPDATE `cloud`.`alert` SET type = 33 WHERE name = 'ALERT.VR.PUBLIC.IFACE.MTU'; +UPDATE `cloud`.`alert` SET type = 34 WHERE name = 'ALERT.VR.PRIVATE.IFACE.MTU'; + +-- Update configuration 'kvm.ssh.to.agent' description and is_dynamic fields +UPDATE `cloud`.`configuration` SET description = 'True if the management server will restart the agent service via SSH into the KVM hosts after or during maintenance operations', is_dynamic = 1 WHERE name = 'kvm.ssh.to.agent'; diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql b/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql index b523016aa3d..858c46a7c1e 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql @@ -87,3 +87,8 @@ CALL `cloud`.`INSERT_EXTENSION_DETAIL_IF_NOT_EXISTS`('MaaS', 'orchestratorrequir CALL `cloud`.`IDEMPOTENT_DROP_UNIQUE_KEY`('counter', 'uc_counter__provider__source__value'); CALL `cloud`.`IDEMPOTENT_ADD_UNIQUE_KEY`('cloud.counter', 'uc_counter__provider__source__value__removed', '(provider, source, value, removed)'); + +-- Change scope for configuration - 'use.https.to.upload from' from StoragePool to Zone +UPDATE `cloud`.`configuration` SET `scope` = 2 WHERE `name` = 'use.https.to.upload'; +-- Delete the configuration for 'use.https.to.upload' from StoragePool +DELETE FROM `cloud`.`storage_pool_details` WHERE `name` = 'use.https.to.upload'; diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42200to42210.sql b/engine/schema/src/main/resources/META-INF/db/schema-42200to42210.sql index 6aafa7ba81e..a8a3d3f7bd4 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42200to42210.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42200to42210.sql @@ -25,3 +25,11 @@ CALL `cloud_usage`.`IDEMPOTENT_ADD_COLUMN`('cloud_usage.usage_event','vm_id', 'b -- Add vm_id column to cloud_usage.usage_volume table CALL `cloud_usage`.`IDEMPOTENT_ADD_COLUMN`('cloud_usage.usage_volume','vm_id', 'bigint UNSIGNED NULL COMMENT "VM ID associated with the volume usage"'); + +ALTER TABLE `cloud`.`template_store_ref` MODIFY COLUMN `download_url` varchar(2048); + +UPDATE `cloud`.`alert` SET type = 33 WHERE name = 'ALERT.VR.PUBLIC.IFACE.MTU'; +UPDATE `cloud`.`alert` SET type = 34 WHERE name = 'ALERT.VR.PRIVATE.IFACE.MTU'; + +-- Update configuration 'kvm.ssh.to.agent' description and is_dynamic fields +UPDATE `cloud`.`configuration` SET description = 'True if the management server will restart the agent service via SSH into the KVM hosts after or during maintenance operations', is_dynamic = 1 WHERE name = 'kvm.ssh.to.agent'; diff --git a/engine/schema/src/test/java/com/cloud/network/dao/NetworkDaoImplTest.java b/engine/schema/src/test/java/com/cloud/network/dao/NetworkDaoImplTest.java index ab5f4352105..a78eab568af 100644 --- a/engine/schema/src/test/java/com/cloud/network/dao/NetworkDaoImplTest.java +++ b/engine/schema/src/test/java/com/cloud/network/dao/NetworkDaoImplTest.java @@ -22,7 +22,6 @@ package com.cloud.network.dao; import com.cloud.network.Networks; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; -import com.cloud.utils.db.TransactionLegacy; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -30,7 +29,6 @@ import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; - import java.util.List; @RunWith(MockitoJUnitRunner.class) @@ -46,26 +44,21 @@ public class NetworkDaoImplTest { List listNetworkVoMock; @Test - public void listByPhysicalNetworkTrafficTypeTestSetParametersValidation() throws Exception { + public void listByPhysicalNetworkTrafficTypeTestSetParametersValidation() { NetworkDaoImpl networkDaoImplSpy = Mockito.spy(NetworkDaoImpl.class); - TransactionLegacy txn = TransactionLegacy.open("runNetworkDaoImplTest"); - try { - networkDaoImplSpy.AllFieldsSearch = searchBuilderNetworkVoMock; - Mockito.doReturn(searchCriteriaNetworkVoMock).when(searchBuilderNetworkVoMock).create(); - Mockito.doNothing().when(searchCriteriaNetworkVoMock).setParameters(Mockito.anyString(), Mockito.any()); - Mockito.doReturn(listNetworkVoMock).when(networkDaoImplSpy).listBy(Mockito.any(SearchCriteria.class)); + networkDaoImplSpy.AllFieldsSearch = searchBuilderNetworkVoMock; + Mockito.doReturn(searchCriteriaNetworkVoMock).when(searchBuilderNetworkVoMock).create(); + Mockito.doNothing().when(searchCriteriaNetworkVoMock).setParameters(Mockito.anyString(), Mockito.any()); + Mockito.doReturn(listNetworkVoMock).when(networkDaoImplSpy).listBy(Mockito.any(SearchCriteria.class)); - long expectedPhysicalNetwork = 2513l; + long expectedPhysicalNetwork = 2513l; - for (Networks.TrafficType trafficType : Networks.TrafficType.values()) { - List result = networkDaoImplSpy.listByPhysicalNetworkTrafficType(expectedPhysicalNetwork, trafficType); - Assert.assertEquals(listNetworkVoMock, result); - Mockito.verify(searchCriteriaNetworkVoMock).setParameters("trafficType", trafficType); - } - - Mockito.verify(searchCriteriaNetworkVoMock, Mockito.times(Networks.TrafficType.values().length)).setParameters("physicalNetwork", expectedPhysicalNetwork); - } finally { - txn.close(); + for (Networks.TrafficType trafficType : Networks.TrafficType.values()) { + List result = networkDaoImplSpy.listByPhysicalNetworkTrafficType(expectedPhysicalNetwork, trafficType); + Assert.assertEquals(listNetworkVoMock, result); + Mockito.verify(searchCriteriaNetworkVoMock).setParameters("trafficType", trafficType); } + + Mockito.verify(searchCriteriaNetworkVoMock, Mockito.times(Networks.TrafficType.values().length)).setParameters("physicalNetworkId", expectedPhysicalNetwork); } } diff --git a/engine/schema/src/test/java/com/cloud/vm/dao/VMInstanceDetailsDaoImplTest.java b/engine/schema/src/test/java/com/cloud/vm/dao/VMInstanceDetailsDaoImplTest.java new file mode 100644 index 00000000000..06238e386d5 --- /dev/null +++ b/engine/schema/src/test/java/com/cloud/vm/dao/VMInstanceDetailsDaoImplTest.java @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.vm.dao; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.vm.VMInstanceDetailVO; + +@RunWith(MockitoJUnitRunner.class) +public class VMInstanceDetailsDaoImplTest { + @Spy + @InjectMocks + private VMInstanceDetailsDaoImpl vmInstanceDetailsDaoImpl; + + @Test + public void removeDetailsWithPrefixReturnsZeroWhenPrefixIsBlank() { + Assert.assertEquals(0, vmInstanceDetailsDaoImpl.removeDetailsWithPrefix(1L, "")); + Assert.assertEquals(0, vmInstanceDetailsDaoImpl.removeDetailsWithPrefix(1L, " ")); + Assert.assertEquals(0, vmInstanceDetailsDaoImpl.removeDetailsWithPrefix(1L, null)); + } + + @Test + public void removeDetailsWithPrefixRemovesMatchingDetails() { + SearchBuilder sb = mock(SearchBuilder.class); + VMInstanceDetailVO entity = mock(VMInstanceDetailVO.class); + when(sb.entity()).thenReturn(entity); + when(sb.and(anyString(), any(), any(SearchCriteria.Op.class))).thenReturn(sb); + SearchCriteria sc = mock(SearchCriteria.class); + when(sb.create()).thenReturn(sc); + when(vmInstanceDetailsDaoImpl.createSearchBuilder()).thenReturn(sb); + doReturn(3).when(vmInstanceDetailsDaoImpl).remove(sc); + int removedCount = vmInstanceDetailsDaoImpl.removeDetailsWithPrefix(1L, "testPrefix"); + Assert.assertEquals(3, removedCount); + Mockito.verify(sc).setParameters("vmId", 1L); + Mockito.verify(sc).setParameters("prefix", "testPrefix%"); + Mockito.verify(vmInstanceDetailsDaoImpl, Mockito.times(1)).remove(sc); + } + + @Test + public void removeDetailsWithPrefixDoesNotRemoveWhenNoMatch() { + SearchBuilder sb = mock(SearchBuilder.class); + VMInstanceDetailVO entity = mock(VMInstanceDetailVO.class); + when(sb.entity()).thenReturn(entity); + when(sb.and(anyString(), any(), any(SearchCriteria.Op.class))).thenReturn(sb); + SearchCriteria sc = mock(SearchCriteria.class); + when(sb.create()).thenReturn(sc); + when(vmInstanceDetailsDaoImpl.createSearchBuilder()).thenReturn(sb); + doReturn(0).when(vmInstanceDetailsDaoImpl).remove(sc); + + int removedCount = vmInstanceDetailsDaoImpl.removeDetailsWithPrefix(1L, "nonExistentPrefix"); + + Assert.assertEquals(0, removedCount); + Mockito.verify(sc).setParameters("vmId", 1L); + Mockito.verify(sc).setParameters("prefix", "nonExistentPrefix%"); + Mockito.verify(vmInstanceDetailsDaoImpl, Mockito.times(1)).remove(sc); + } +} diff --git a/engine/service/pom.xml b/engine/service/pom.xml index 9b833804064..673d9c418d7 100644 --- a/engine/service/pom.xml +++ b/engine/service/pom.xml @@ -69,7 +69,7 @@ org.apache.maven.plugins maven-war-plugin - 3.4.0 + 3.5.1 org.eclipse.jetty diff --git a/engine/storage/configdrive/src/main/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilder.java b/engine/storage/configdrive/src/main/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilder.java index 0b81a25b1cd..15febbe972c 100644 --- a/engine/storage/configdrive/src/main/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilder.java +++ b/engine/storage/configdrive/src/main/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilder.java @@ -231,9 +231,9 @@ public class ConfigDriveBuilder { throw new CloudRuntimeException("Cannot create ISO for config drive using any know tool. Known paths [/usr/bin/genisoimage, /usr/bin/mkisofs, /usr/local/bin/mkisofs]"); } if (!isoCreator.canExecute()) { - throw new CloudRuntimeException("Cannot create ISO for config drive using: " + isoCreator.getCanonicalPath()); + throw new CloudRuntimeException("Cannot create ISO for config drive using: " + isoCreator.getAbsolutePath()); } - return isoCreator.getCanonicalPath(); + return isoCreator.getAbsolutePath(); } /** diff --git a/engine/storage/configdrive/src/test/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilderTest.java b/engine/storage/configdrive/src/test/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilderTest.java index c04ff0a1601..03ceac84399 100644 --- a/engine/storage/configdrive/src/test/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilderTest.java +++ b/engine/storage/configdrive/src/test/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilderTest.java @@ -435,7 +435,7 @@ public class ConfigDriveBuilderTest { Mockito.verify(genIsoFileMock, Mockito.times(2)).exists(); Mockito.verify(genIsoFileMock).canExecute(); - Mockito.verify(genIsoFileMock).getCanonicalPath(); + Mockito.verify(genIsoFileMock).getAbsolutePath(); } } @@ -475,11 +475,11 @@ public class ConfigDriveBuilderTest { Mockito.verify(genIsoFileMock, Mockito.times(1)).exists(); Mockito.verify(genIsoFileMock, Mockito.times(0)).canExecute(); - Mockito.verify(genIsoFileMock, Mockito.times(0)).getCanonicalPath(); + Mockito.verify(genIsoFileMock, Mockito.times(0)).getAbsolutePath(); Mockito.verify(mkIsoProgramInLinuxFileMock, Mockito.times(2)).exists(); Mockito.verify(mkIsoProgramInLinuxFileMock, Mockito.times(1)).canExecute(); - Mockito.verify(mkIsoProgramInLinuxFileMock, Mockito.times(1)).getCanonicalPath(); + Mockito.verify(mkIsoProgramInLinuxFileMock, Mockito.times(1)).getAbsolutePath(); } } @@ -509,15 +509,15 @@ public class ConfigDriveBuilderTest { Mockito.verify(genIsoFileMock, Mockito.times(1)).exists(); Mockito.verify(genIsoFileMock, Mockito.times(0)).canExecute(); - Mockito.verify(genIsoFileMock, Mockito.times(0)).getCanonicalPath(); + Mockito.verify(genIsoFileMock, Mockito.times(0)).getAbsolutePath(); Mockito.verify(mkIsoProgramInLinuxFileMock, Mockito.times(1)).exists(); Mockito.verify(mkIsoProgramInLinuxFileMock, Mockito.times(0)).canExecute(); - Mockito.verify(mkIsoProgramInLinuxFileMock, Mockito.times(0)).getCanonicalPath(); + Mockito.verify(mkIsoProgramInLinuxFileMock, Mockito.times(0)).getAbsolutePath(); Mockito.verify(mkIsoProgramInMacOsFileMock, Mockito.times(1)).exists(); Mockito.verify(mkIsoProgramInMacOsFileMock, Mockito.times(1)).canExecute(); - Mockito.verify(mkIsoProgramInMacOsFileMock, Mockito.times(1)).getCanonicalPath(); + Mockito.verify(mkIsoProgramInMacOsFileMock, Mockito.times(1)).getAbsolutePath(); } } diff --git a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java index 6b8f4715482..8145158dfa4 100644 --- a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java +++ b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java @@ -47,10 +47,12 @@ import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; import org.apache.cloudstack.framework.async.AsyncCompletionCallback; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.secstorage.heuristics.HeuristicType; import org.apache.cloudstack.storage.RemoteHostEndPoint; import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao; import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO; +import org.apache.cloudstack.storage.heuristics.HeuristicRuleHelper; import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; import org.apache.logging.log4j.Logger; @@ -112,6 +114,9 @@ public class AncientDataMotionStrategy implements DataMotionStrategy { @Inject SnapshotDao snapshotDao; + @Inject + HeuristicRuleHelper heuristicRuleHelper; + @Override public StrategyPriority canHandle(DataObject srcData, DataObject destData) { return StrategyPriority.DEFAULT; @@ -379,7 +384,13 @@ public class AncientDataMotionStrategy implements DataMotionStrategy { } // need to find a nfs or cifs image store, assuming that can't copy volume // directly to s3 - ImageStoreEntity imageStore = (ImageStoreEntity)dataStoreMgr.getImageStoreWithFreeCapacity(destScope.getScopeId()); + Long zoneId = destScope.getScopeId(); + ImageStoreEntity imageStore = (ImageStoreEntity) heuristicRuleHelper.getImageStoreIfThereIsHeuristicRule(zoneId, HeuristicType.VOLUME, destData); + if (imageStore == null) { + logger.debug("Secondary storage selector did not direct volume migration to a specific secondary storage; using secondary storage with the most free capacity."); + imageStore = (ImageStoreEntity) dataStoreMgr.getImageStoreWithFreeCapacity(zoneId); + } + if (imageStore == null || !imageStore.getProtocol().equalsIgnoreCase("nfs") && !imageStore.getProtocol().equalsIgnoreCase("cifs")) { String errMsg = "can't find a nfs (or cifs) image store to satisfy the need for a staging store"; Answer answer = new Answer(null, false, errMsg); diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/SecondaryStorageServiceImpl.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/SecondaryStorageServiceImpl.java index 641a2a40dcd..f739fecf9bf 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/SecondaryStorageServiceImpl.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/SecondaryStorageServiceImpl.java @@ -280,7 +280,7 @@ public class SecondaryStorageServiceImpl implements SecondaryStorageService { private void updateDataObject(DataObject srcData, DataObject destData) { if (destData instanceof SnapshotInfo) { SnapshotDataStoreVO snapshotStore = snapshotStoreDao.findBySourceSnapshot(srcData.getId(), DataStoreRole.Image); - SnapshotDataStoreVO destSnapshotStore = snapshotStoreDao.findByStoreSnapshot(DataStoreRole.Image, srcData.getDataStore().getId(), srcData.getId()); + SnapshotDataStoreVO destSnapshotStore = snapshotStoreDao.findByStoreSnapshot(DataStoreRole.Image, destData.getDataStore().getId(), destData.getId()); if (snapshotStore != null && destSnapshotStore != null) { destSnapshotStore.setPhysicalSize(snapshotStore.getPhysicalSize()); destSnapshotStore.setCreated(snapshotStore.getCreated()); diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java index c6430bcf9f9..3e1504beb3a 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java @@ -296,6 +296,9 @@ public class TemplateDataFactoryImpl implements TemplateDataFactory { @Override public boolean isTemplateMarkedForDirectDownload(long templateId) { VMTemplateVO templateVO = imageDataDao.findById(templateId); + if (templateVO == null) { + throw new CloudRuntimeException(String.format("Template not found with ID: %s", templateId)); + } return templateVO.isDirectDownload(); } } diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java index 4b82bedefeb..e29e89cf431 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java @@ -31,6 +31,8 @@ import java.util.concurrent.ExecutionException; import javax.inject.Inject; +import com.cloud.exception.StorageUnavailableException; +import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.orchestration.service.StorageOrchestrationService; import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult; @@ -67,9 +69,11 @@ import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; import org.apache.cloudstack.storage.image.store.TemplateObject; import org.apache.cloudstack.storage.to.TemplateObjectTO; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.ThreadContext; import org.springframework.stereotype.Component; import com.cloud.agent.api.Answer; @@ -291,21 +295,41 @@ public class TemplateServiceImpl implements TemplateService { } } - protected boolean isSkipTemplateStoreDownload(VMTemplateVO template, Long zoneId) { + protected boolean shouldDownloadTemplateToStore(VMTemplateVO template, DataStore store) { + Long zoneId = store.getScope().getScopeId(); + DataStore directedStore = _tmpltMgr.verifyHeuristicRulesForZone(template, zoneId); + if (directedStore != null && store.getId() != directedStore.getId()) { + logger.info("Template [{}] will not be download to image store [{}], as a heuristic rule is directing it to another store.", + template.getUniqueName(), store.getName()); + return false; + } + if (template.isPublicTemplate()) { - return false; + logger.debug("Download of template [{}] to image store [{}] cannot be skipped, as it is public.", template.getUniqueName(), + store.getName()); + return true; } + if (template.isFeatured()) { - return false; + logger.debug("Download of template [{}] to image store [{}] cannot be skipped, as it is featured.", template.getUniqueName(), + store.getName()); + return true; } + if (TemplateType.SYSTEM.equals(template.getTemplateType())) { - return false; + logger.debug("Download of template [{}] to image store [{}] cannot be skipped, as it is a system VM template.", + template.getUniqueName(),store.getName()); + return true; } + if (zoneId != null && _vmTemplateStoreDao.findByTemplateZone(template.getId(), zoneId, DataStoreRole.Image) == null) { - logger.debug("Template {} is not present on any image store for the zone ID: {}, its download cannot be skipped", template, zoneId); - return false; + logger.debug("Download of template [{}] to image store [{}] cannot be skipped, as it is not present on any image store of zone [{}].", + template.getUniqueName(), store.getName(), zoneId); + return true; } - return true; + + logger.info("Skipping download of template [{}] to image store [{}].", template.getUniqueName(), store.getName()); + return false; } @Override @@ -533,8 +557,7 @@ public class TemplateServiceImpl implements TemplateService { // download. for (VMTemplateVO tmplt : toBeDownloaded) { // if this is private template, skip sync to a new image store - if (isSkipTemplateStoreDownload(tmplt, zoneId)) { - logger.info("Skip sync downloading private template {} to a new image store", tmplt); + if (!shouldDownloadTemplateToStore(tmplt, store)) { continue; } @@ -550,10 +573,7 @@ public class TemplateServiceImpl implements TemplateService { } if (availHypers.contains(tmplt.getHypervisorType())) { - boolean copied = isCopyFromOtherStoragesEnabled(zoneId) && tryCopyingTemplateToImageStore(tmplt, store); - if (!copied) { - tryDownloadingTemplateToImageStore(tmplt, store); - } + storageOrchestrator.orchestrateTemplateCopyFromSecondaryStores(tmplt.getId(), store); } else { logger.info("Skip downloading template {} since current data center does not have hypervisor {}", tmplt, tmplt.getHypervisorType()); } @@ -600,6 +620,16 @@ public class TemplateServiceImpl implements TemplateService { } + @Override + public void handleTemplateCopyFromSecondaryStores(long templateId, DataStore destStore) { + VMTemplateVO template = _templateDao.findById(templateId); + long zoneId = destStore.getScope().getScopeId(); + boolean copied = imageStoreDetailsUtil.isCopyTemplatesFromOtherStoragesEnabled(destStore.getId(), zoneId) && tryCopyingTemplateToImageStore(template, destStore); + if (!copied) { + tryDownloadingTemplateToImageStore(template, destStore); + } + } + protected void tryDownloadingTemplateToImageStore(VMTemplateVO tmplt, DataStore destStore) { if (tmplt.getUrl() == null) { logger.info("Not downloading template [{}] to image store [{}], as it has no URL.", tmplt.getUniqueName(), @@ -617,28 +647,134 @@ public class TemplateServiceImpl implements TemplateService { } protected boolean tryCopyingTemplateToImageStore(VMTemplateVO tmplt, DataStore destStore) { - Long zoneId = destStore.getScope().getScopeId(); - List storesInZone = _storeMgr.getImageStoresByZoneIds(zoneId); - for (DataStore sourceStore : storesInZone) { - Map existingTemplatesInSourceStore = listTemplate(sourceStore); - if (existingTemplatesInSourceStore == null || !existingTemplatesInSourceStore.containsKey(tmplt.getUniqueName())) { - logger.debug("Template [{}] does not exist on image store [{}]; searching on another one.", - tmplt.getUniqueName(), sourceStore.getName()); - continue; - } - TemplateObject sourceTmpl = (TemplateObject) _templateFactory.getTemplate(tmplt.getId(), sourceStore); - if (sourceTmpl.getInstallPath() == null) { - logger.warn("Can not copy template [{}] from image store [{}], as it returned a null install path.", tmplt.getUniqueName(), - sourceStore.getName()); - continue; - } - storageOrchestrator.orchestrateTemplateCopyToImageStore(sourceTmpl, destStore); + if (searchAndCopyWithinZone(tmplt, destStore)) { return true; } - logger.debug("Can't copy template [{}] from another image store.", tmplt.getUniqueName()); + + Long destZoneId = destStore.getScope().getScopeId(); + logger.debug("Template [{}] not found in any image store of zone [{}]. Checking other zones.", + tmplt.getUniqueName(), destZoneId); + + return searchAndCopyAcrossZones(tmplt, destStore, destZoneId); + } + + private boolean searchAndCopyAcrossZones(VMTemplateVO tmplt, DataStore destStore, Long destZoneId) { + List allZoneIds = _dcDao.listAllIds(); + for (Long otherZoneId : allZoneIds) { + if (otherZoneId.equals(destZoneId)) { + continue; + } + + List storesInOtherZone = _storeMgr.getImageStoresByZoneIds(otherZoneId); + logger.debug("Checking zone [{}] for template [{}]...", otherZoneId, tmplt.getUniqueName()); + + if (CollectionUtils.isEmpty(storesInOtherZone)) { + logger.debug("Zone [{}] has no image stores. Skipping.", otherZoneId); + continue; + } + + TemplateObject sourceTmpl = findUsableTemplate(tmplt, storesInOtherZone); + if (sourceTmpl == null) { + logger.debug("Template [{}] not found with a valid install path in any image store of zone [{}].", + tmplt.getUniqueName(), otherZoneId); + continue; + } + + logger.info("Template [{}] found in zone [{}]. Initiating cross-zone copy to zone [{}].", + tmplt.getUniqueName(), otherZoneId, destZoneId); + + return copyTemplateAcrossZones(destStore, sourceTmpl); + } + + logger.debug("Template [{}] was not found in any zone. Cannot perform zone-to-zone copy.", tmplt.getUniqueName()); return false; } + protected TemplateObject findUsableTemplate(VMTemplateVO tmplt, List imageStores) { + for (DataStore store : imageStores) { + + Map templates = listTemplate(store); + if (templates == null || !templates.containsKey(tmplt.getUniqueName())) { + continue; + } + + TemplateObject tmpl = (TemplateObject) _templateFactory.getTemplate(tmplt.getId(), store); + if (tmpl.getInstallPath() == null) { + logger.debug("Template [{}] found in image store [{}] but install path is null. Skipping.", + tmplt.getUniqueName(), store.getName()); + continue; + } + return tmpl; + } + return null; + } + + private boolean searchAndCopyWithinZone(VMTemplateVO tmplt, DataStore destStore) { + Long destZoneId = destStore.getScope().getScopeId(); + List storesInSameZone = _storeMgr.getImageStoresByZoneIds(destZoneId); + + TemplateObject sourceTmpl = findUsableTemplate(tmplt, storesInSameZone); + if (sourceTmpl == null) { + return false; + } + + TemplateApiResult result; + AsyncCallFuture future = copyTemplateToImageStore(sourceTmpl, destStore); + try { + result = future.get(); + } catch (ExecutionException | InterruptedException e) { + logger.warn("Exception while copying template [{}] from image store [{}] to image store [{}]: {}", + sourceTmpl.getUniqueName(), sourceTmpl.getDataStore().getName(), destStore.getName(), e.toString()); + result = new TemplateApiResult(sourceTmpl); + result.setResult(e.getMessage()); + } + return result.isSuccess(); + } + + private boolean copyTemplateAcrossZones(DataStore destStore, TemplateObject sourceTmpl) { + Long dstZoneId = destStore.getScope().getScopeId(); + DataCenterVO dstZone = _dcDao.findById(dstZoneId); + + if (dstZone == null) { + logger.warn("Destination zone [{}] not found for template [{}].", dstZoneId, sourceTmpl.getUniqueName()); + return false; + } + + TemplateApiResult result; + try { + VMTemplateVO template = _templateDao.findById(sourceTmpl.getId()); + try { + DataStore sourceStore = sourceTmpl.getDataStore(); + long userId = CallContext.current().getCallingUserId(); + boolean success = _tmpltMgr.copy(userId, template, sourceStore, dstZone); + + result = new TemplateApiResult(sourceTmpl); + if (!success) { + result.setResult("Cross-zone template copy failed"); + } + } catch (StorageUnavailableException | ResourceAllocationException e) { + logger.error("Exception while copying template [{}] from zone [{}] to zone [{}]", + template, + sourceTmpl.getDataStore().getScope().getScopeId(), + dstZone.getId(), + e); + result = new TemplateApiResult(sourceTmpl); + result.setResult(e.getMessage()); + } finally { + ThreadContext.clearAll(); + } + } catch (Exception e) { + logger.error("Failed to copy template [{}] from zone [{}] to zone [{}].", + sourceTmpl.getUniqueName(), + sourceTmpl.getDataStore().getScope().getScopeId(), + dstZoneId, + e); + return false; + } + + return result.isSuccess(); + } + @Override public AsyncCallFuture copyTemplateToImageStore(DataObject source, DataStore destStore) { TemplateObject sourceTmpl = (TemplateObject) source; @@ -682,10 +818,6 @@ public class TemplateServiceImpl implements TemplateService { return null; } - protected boolean isCopyFromOtherStoragesEnabled(Long zoneId) { - return StorageManager.COPY_PUBLIC_TEMPLATES_FROM_OTHER_STORAGES.valueIn(zoneId); - } - protected void publishTemplateCreation(TemplateInfo tmplt) { VMTemplateVO tmpltVo = _templateDao.findById(tmplt.getId()); @@ -1320,9 +1452,10 @@ public class TemplateServiceImpl implements TemplateService { if (_vmTemplateStoreDao.isTemplateMarkedForDirectDownload(tmplt.getId())) { continue; } - tmpltStore = - new TemplateDataStoreVO(storeId, tmplt.getId(), new Date(), 100, Status.DOWNLOADED, null, null, null, - TemplateConstants.DEFAULT_SYSTEM_VM_TEMPLATE_PATH + tmplt.getId() + '/', tmplt.getUrl()); + String templateDirectoryPath = TemplateConstants.DEFAULT_TMPLT_ROOT_DIR + File.separator + TemplateConstants.DEFAULT_TMPLT_FIRST_LEVEL_DIR; + String installPath = templateDirectoryPath + tmplt.getAccountId() + File.separator + tmplt.getId() + File.separator; + tmpltStore = new TemplateDataStoreVO(storeId, tmplt.getId(), new Date(), 100, Status.DOWNLOADED, + null, null, null, installPath, tmplt.getUrl()); tmpltStore.setSize(0L); tmpltStore.setPhysicalSize(0); // no size information for // pre-seeded system vm templates diff --git a/engine/storage/image/src/test/java/org/apache/cloudstack/storage/image/TemplateServiceImplTest.java b/engine/storage/image/src/test/java/org/apache/cloudstack/storage/image/TemplateServiceImplTest.java index 276581e2e48..e9eac045869 100644 --- a/engine/storage/image/src/test/java/org/apache/cloudstack/storage/image/TemplateServiceImplTest.java +++ b/engine/storage/image/src/test/java/org/apache/cloudstack/storage/image/TemplateServiceImplTest.java @@ -18,12 +18,20 @@ */ package org.apache.cloudstack.storage.image; +import com.cloud.dc.DataCenterVO; +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.StorageUnavailableException; +import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.template.TemplateProp; +import com.cloud.template.TemplateManager; +import com.cloud.user.Account; +import com.cloud.user.User; +import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.orchestration.service.StorageOrchestrationService; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.Scope; -import org.apache.cloudstack.framework.async.AsyncCallFuture; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import org.apache.cloudstack.storage.image.store.TemplateObject; @@ -45,6 +53,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import static org.mockito.Mockito.mock; + @RunWith(MockitoJUnitRunner.class) public class TemplateServiceImplTest { @@ -70,6 +80,9 @@ public class TemplateServiceImplTest { @Mock TemplateObject templateInfoMock; + @Mock + DataStore dataStoreMock; + @Mock DataStore sourceStoreMock; @@ -82,6 +95,15 @@ public class TemplateServiceImplTest { @Mock StorageOrchestrationService storageOrchestrator; + @Mock + TemplateManager templateManagerMock; + + @Mock + VMTemplateDao templateDao; + + @Mock + DataCenterDao _dcDao; + Map templatesInSourceStore = new HashMap<>(); @Before @@ -94,47 +116,46 @@ public class TemplateServiceImplTest { Mockito.doReturn(List.of(sourceStoreMock, destStoreMock)).when(dataStoreManagerMock).getImageStoresByZoneIds(zoneId); Mockito.doReturn(templatesInSourceStore).when(templateService).listTemplate(sourceStoreMock); Mockito.doReturn(null).when(templateService).listTemplate(destStoreMock); - Mockito.doReturn("install-path").when(templateInfoMock).getInstallPath(); Mockito.doReturn(templateInfoMock).when(templateDataFactoryMock).getTemplate(2L, sourceStoreMock); + Mockito.doReturn(3L).when(dataStoreMock).getId(); + Mockito.doReturn(zoneScopeMock).when(dataStoreMock).getScope(); } @Test - public void testIsSkipTemplateStoreDownloadPublicTemplate() { - VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class); - Mockito.when(templateVO.isPublicTemplate()).thenReturn(true); - Assert.assertFalse(templateService.isSkipTemplateStoreDownload(templateVO, 1L)); + public void shouldDownloadTemplateToStoreTestSkipsTemplateDirectedToAnotherStorage() { + DataStore destinedStore = Mockito.mock(DataStore.class); + Mockito.doReturn(dataStoreMock.getId() + 1L).when(destinedStore).getId(); + Mockito.when(templateManagerMock.verifyHeuristicRulesForZone(tmpltMock, zoneScopeMock.getScopeId())).thenReturn(destinedStore); + Assert.assertFalse(templateService.shouldDownloadTemplateToStore(tmpltMock, dataStoreMock)); } @Test - public void testIsSkipTemplateStoreDownloadFeaturedTemplate() { - VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class); - Mockito.when(templateVO.isFeatured()).thenReturn(true); - Assert.assertFalse(templateService.isSkipTemplateStoreDownload(templateVO, 1L)); + public void shouldDownloadTemplateToStoreTestDownloadsPublicTemplate() { + Mockito.when(tmpltMock.isPublicTemplate()).thenReturn(true); + Assert.assertTrue(templateService.shouldDownloadTemplateToStore(tmpltMock, dataStoreMock)); } @Test - public void testIsSkipTemplateStoreDownloadSystemTemplate() { - VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class); - Mockito.when(templateVO.getTemplateType()).thenReturn(Storage.TemplateType.SYSTEM); - Assert.assertFalse(templateService.isSkipTemplateStoreDownload(templateVO, 1L)); + public void shouldDownloadTemplateToStoreTestDownloadsFeaturedTemplate() { + Mockito.when(tmpltMock.isFeatured()).thenReturn(true); + Assert.assertTrue(templateService.shouldDownloadTemplateToStore(tmpltMock, dataStoreMock)); } @Test - public void testIsSkipTemplateStoreDownloadPrivateNoRefTemplate() { - VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class); - long id = 1L; - Mockito.when(templateVO.getId()).thenReturn(id); - Mockito.when(templateDataStoreDao.findByTemplateZone(id, id, DataStoreRole.Image)).thenReturn(null); - Assert.assertFalse(templateService.isSkipTemplateStoreDownload(templateVO, id)); + public void shouldDownloadTemplateToStoreTestDownloadsSystemTemplate() { + Mockito.when(tmpltMock.getTemplateType()).thenReturn(Storage.TemplateType.SYSTEM); + Assert.assertTrue(templateService.shouldDownloadTemplateToStore(tmpltMock, dataStoreMock)); } @Test - public void testIsSkipTemplateStoreDownloadPrivateExistingTemplate() { - VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class); - long id = 1L; - Mockito.when(templateVO.getId()).thenReturn(id); - Mockito.when(templateDataStoreDao.findByTemplateZone(id, id, DataStoreRole.Image)).thenReturn(Mockito.mock(TemplateDataStoreVO.class)); - Assert.assertTrue(templateService.isSkipTemplateStoreDownload(templateVO, id)); + public void shouldDownloadTemplateToStoreTestDownloadsPrivateNoRefTemplate() { + Assert.assertTrue(templateService.shouldDownloadTemplateToStore(tmpltMock, dataStoreMock)); + } + + @Test + public void shouldDownloadTemplateToStoreTestSkipsPrivateExistingTemplate() { + Mockito.when(templateDataStoreDao.findByTemplateZone(tmpltMock.getId(), zoneScopeMock.getScopeId(), DataStoreRole.Image)).thenReturn(Mockito.mock(TemplateDataStoreVO.class)); + Assert.assertFalse(templateService.shouldDownloadTemplateToStore(tmpltMock, dataStoreMock)); } @Test @@ -159,7 +180,7 @@ public class TemplateServiceImplTest { boolean result = templateService.tryCopyingTemplateToImageStore(tmpltMock, destStoreMock); Assert.assertFalse(result); - Mockito.verify(storageOrchestrator, Mockito.never()).orchestrateTemplateCopyToImageStore(Mockito.any(), Mockito.any()); + Mockito.verify(storageOrchestrator, Mockito.never()).orchestrateTemplateCopyFromSecondaryStores(Mockito.anyLong(), Mockito.any()); } @Test @@ -167,20 +188,161 @@ public class TemplateServiceImplTest { templatesInSourceStore.put(tmpltMock.getUniqueName(), tmpltPropMock); Mockito.doReturn(null).when(templateInfoMock).getInstallPath(); + Scope scopeMock = Mockito.mock(Scope.class); + Mockito.doReturn(scopeMock).when(destStoreMock).getScope(); + Mockito.doReturn(1L).when(scopeMock).getScopeId(); + Mockito.doReturn(List.of(1L)).when(_dcDao).listAllIds(); + boolean result = templateService.tryCopyingTemplateToImageStore(tmpltMock, destStoreMock); Assert.assertFalse(result); - Mockito.verify(storageOrchestrator, Mockito.never()).orchestrateTemplateCopyToImageStore(Mockito.any(), Mockito.any()); + Mockito.verify(storageOrchestrator, Mockito.never()).orchestrateTemplateCopyFromSecondaryStores(Mockito.anyLong(), Mockito.any()); } @Test - public void tryCopyingTemplateToImageStoreTestReturnsTrueWhenTemplateExistsInAnotherStorageAndTaskWasScheduled() { - templatesInSourceStore.put(tmpltMock.getUniqueName(), tmpltPropMock); - Mockito.doReturn(new AsyncCallFuture<>()).when(storageOrchestrator).orchestrateTemplateCopyToImageStore(Mockito.any(), Mockito.any()); + public void tryCopyingTemplateToImageStoreTestReturnsTrueWhenTemplateExistsInAnotherZone() throws StorageUnavailableException, ResourceAllocationException { + Scope scopeMock = Mockito.mock(Scope.class); + Mockito.doReturn(scopeMock).when(destStoreMock).getScope(); + Mockito.doReturn(1L).when(scopeMock).getScopeId(); + Mockito.doReturn(100L).when(tmpltMock).getId(); + Mockito.doReturn("unique-name").when(tmpltMock).getUniqueName(); + Mockito.doReturn(List.of(sourceStoreMock)).when(dataStoreManagerMock).getImageStoresByZoneIds(1L); + Mockito.doReturn(null).when(templateService).listTemplate(sourceStoreMock); + Mockito.doReturn(List.of(1L, 2L)).when(_dcDao).listAllIds(); + + DataStore otherZoneStoreMock = Mockito.mock(DataStore.class); + Mockito.doReturn(List.of(otherZoneStoreMock)).when(dataStoreManagerMock).getImageStoresByZoneIds(2L); + + Map templatesInOtherZone = new HashMap<>(); + templatesInOtherZone.put("unique-name", tmpltPropMock); + Mockito.doReturn(templatesInOtherZone).when(templateService).listTemplate(otherZoneStoreMock); + + TemplateObject sourceTmplMock = Mockito.mock(TemplateObject.class); + Mockito.doReturn(sourceTmplMock).when(templateDataFactoryMock).getTemplate(100L, otherZoneStoreMock); + Mockito.doReturn("/mnt/secondary/template.qcow2").when(sourceTmplMock).getInstallPath(); + + DataCenterVO dstZoneMock = Mockito.mock(DataCenterVO.class); + Mockito.doReturn(dstZoneMock).when(_dcDao).findById(1L); + Mockito.doReturn(true).when(templateManagerMock).copy(Mockito.anyLong(), Mockito.any(), Mockito.any(), Mockito.any()); boolean result = templateService.tryCopyingTemplateToImageStore(tmpltMock, destStoreMock); Assert.assertTrue(result); - Mockito.verify(storageOrchestrator).orchestrateTemplateCopyToImageStore(Mockito.any(), Mockito.any()); + } + + @Test + public void tryCopyingTemplateToImageStoreTestReturnsFalseWhenDestinationZoneIsMissing() { + Scope scopeMock = Mockito.mock(Scope.class); + Mockito.doReturn(scopeMock).when(destStoreMock).getScope(); + Mockito.doReturn(1L).when(scopeMock).getScopeId(); + Mockito.doReturn(100L).when(tmpltMock).getId(); + Mockito.doReturn("unique-name").when(tmpltMock).getUniqueName(); + Mockito.doReturn(List.of(1L, 2L)).when(_dcDao).listAllIds(); + Mockito.doReturn(List.of()).when(dataStoreManagerMock).getImageStoresByZoneIds(1L); + + DataStore otherZoneStoreMock = Mockito.mock(DataStore.class); + Mockito.doReturn(List.of(otherZoneStoreMock)).when(dataStoreManagerMock).getImageStoresByZoneIds(2L); + + Map templates = new HashMap<>(); + templates.put("unique-name", tmpltPropMock); + Mockito.doReturn(templates).when(templateService).listTemplate(otherZoneStoreMock); + + TemplateObject sourceTmplMock = Mockito.mock(TemplateObject.class); + Mockito.doReturn(sourceTmplMock).when(templateDataFactoryMock).getTemplate(100L, otherZoneStoreMock); + Mockito.doReturn("/mnt/secondary/template.qcow2").when(sourceTmplMock).getInstallPath(); + Mockito.doReturn(null).when(_dcDao).findById(1L); + + boolean result = templateService.tryCopyingTemplateToImageStore(tmpltMock, destStoreMock); + + Assert.assertFalse(result); + } + + @Test + public void tryCopyingTemplateToImageStoreTestReturnsTrueWhenCrossZoneCopyTaskIsScheduled() throws StorageUnavailableException, ResourceAllocationException { + Scope scopeMock = Mockito.mock(Scope.class); + Mockito.doReturn(scopeMock).when(destStoreMock).getScope(); + Mockito.doReturn(1L).when(scopeMock).getScopeId(); + Mockito.doReturn(100L).when(tmpltMock).getId(); + Mockito.doReturn("unique-name").when(tmpltMock).getUniqueName(); + Mockito.doReturn(List.of(1L, 2L)).when(_dcDao).listAllIds(); + Mockito.doReturn(List.of()).when(dataStoreManagerMock).getImageStoresByZoneIds(1L); + + DataStore otherZoneStoreMock = Mockito.mock(DataStore.class); + Mockito.doReturn(List.of(otherZoneStoreMock)).when(dataStoreManagerMock).getImageStoresByZoneIds(2L); + + Map templates = new HashMap<>(); + templates.put("unique-name", tmpltPropMock); + Mockito.doReturn(templates).when(templateService).listTemplate(otherZoneStoreMock); + + TemplateObject sourceTmplMock = Mockito.mock(TemplateObject.class); + Mockito.doReturn(sourceTmplMock).when(templateDataFactoryMock).getTemplate(100L, otherZoneStoreMock); + Mockito.doReturn("/mnt/secondary/template.qcow2").when(sourceTmplMock).getInstallPath(); + Mockito.doReturn(100L).when(sourceTmplMock).getId(); + + DataStore sourceStoreMock = Mockito.mock(DataStore.class); + Scope sourceScopeMock = Mockito.mock(Scope.class); + Mockito.doReturn(sourceStoreMock).when(sourceTmplMock).getDataStore(); + + DataCenterVO dstZoneMock = Mockito.mock(DataCenterVO.class); + Mockito.doReturn(dstZoneMock).when(_dcDao).findById(1L); + VMTemplateVO templateVoMock = Mockito.mock(VMTemplateVO.class); + Mockito.doReturn(templateVoMock).when(templateDao).findById(100L); + + Mockito.doReturn(true).when(templateManagerMock).copy(Mockito.anyLong(), Mockito.any(), Mockito.any(), Mockito.any()); + + Account account = mock(Account.class); + User user = mock(User.class); + CallContext callContext = mock(CallContext.class); + + boolean result = templateService.tryCopyingTemplateToImageStore(tmpltMock, destStoreMock); + + Assert.assertTrue(result); + } + + @Test + public void tryCopyingTemplateToImageStoreTestReturnsFalseWhenTemplateNotFoundInAnyZone() { + Scope scopeMock = Mockito.mock(Scope.class); + Mockito.doReturn(scopeMock).when(destStoreMock).getScope(); + Mockito.doReturn(1L).when(scopeMock).getScopeId(); + Mockito.doReturn(List.of(1L, 2L)).when(_dcDao).listAllIds(); + Mockito.doReturn(List.of(sourceStoreMock)).when(dataStoreManagerMock).getImageStoresByZoneIds(Mockito.anyLong()); + Mockito.doReturn(null).when(templateService).listTemplate(Mockito.any()); + + boolean result = templateService.tryCopyingTemplateToImageStore(tmpltMock, destStoreMock); + + Assert.assertFalse(result); + } + + @Test + public void testFindUsableTemplateReturnsTemplateWithNonNullInstallPath() { + VMTemplateVO template = Mockito.mock(VMTemplateVO.class); + Mockito.when(template.getId()).thenReturn(10L); + Mockito.when(template.getUniqueName()).thenReturn("test-template"); + + DataStore storeWithNullPath = Mockito.mock(DataStore.class); + Mockito.when(storeWithNullPath.getName()).thenReturn("store-null"); + + DataStore storeWithValidPath = Mockito.mock(DataStore.class); + TemplateObject tmplWithNullPath = Mockito.mock(TemplateObject.class); + Mockito.when(tmplWithNullPath.getInstallPath()).thenReturn(null); + + TemplateObject tmplWithValidPath = Mockito.mock(TemplateObject.class); + Mockito.when(tmplWithValidPath.getInstallPath()).thenReturn("/mnt/secondary/template.qcow2"); + + Mockito.doReturn(tmplWithNullPath).when(templateDataFactoryMock).getTemplate(10L, storeWithNullPath); + Mockito.doReturn(tmplWithValidPath).when(templateDataFactoryMock).getTemplate(10L, storeWithValidPath); + + Map templates = new HashMap<>(); + templates.put("test-template", Mockito.mock(TemplateProp.class)); + + Mockito.doReturn(templates).when(templateService).listTemplate(storeWithNullPath); + Mockito.doReturn(templates).when(templateService).listTemplate(storeWithValidPath); + + List imageStores = List.of(storeWithNullPath, storeWithValidPath); + + TemplateObject result = templateService.findUsableTemplate(template, imageStores); + + Assert.assertNotNull(result); + Assert.assertEquals(tmplWithValidPath, result); } } diff --git a/engine/storage/snapshot/pom.xml b/engine/storage/snapshot/pom.xml index 177ab1069d9..0966082b45e 100644 --- a/engine/storage/snapshot/pom.xml +++ b/engine/storage/snapshot/pom.xml @@ -57,8 +57,8 @@ compile - mysql - mysql-connector-java + com.mysql + mysql-connector-j test diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotObject.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotObject.java index 8264fcd4286..6a8bbd93ca4 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotObject.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotObject.java @@ -398,13 +398,16 @@ public class SnapshotObject implements SnapshotInfo { if (answer instanceof CreateObjectAnswer) { SnapshotObjectTO snapshotTO = (SnapshotObjectTO)((CreateObjectAnswer)answer).getData(); snapshotStore.setInstallPath(snapshotTO.getPath()); + if (snapshotTO.getPhysicalSize() != null && snapshotTO.getPhysicalSize() > 0L) { + snapshotStore.setPhysicalSize(snapshotTO.getPhysicalSize()); + } snapshotStoreDao.update(snapshotStore.getId(), snapshotStore); } else if (answer instanceof CopyCmdAnswer) { SnapshotObjectTO snapshotTO = (SnapshotObjectTO)((CopyCmdAnswer)answer).getNewData(); snapshotStore.setInstallPath(snapshotTO.getPath()); if (snapshotTO.getPhysicalSize() != null) { // For S3 delta snapshot, physical size is currently not set - snapshotStore.setPhysicalSize(snapshotTO.getPhysicalSize()); + snapshotStore.setPhysicalSize(snapshotTO.getPhysicalSize()); } if (snapshotTO.getParentSnapshotPath() == null) { snapshotStore.setParentSnapshotId(0L); diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/StorageSystemSnapshotStrategy.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/StorageSystemSnapshotStrategy.java index a19397d03e3..560bb4b2fc1 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/StorageSystemSnapshotStrategy.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/StorageSystemSnapshotStrategy.java @@ -951,7 +951,7 @@ public class StorageSystemSnapshotStrategy extends SnapshotStrategyBase { VolumeVO volumeVO = volumeDao.findByIdIncludingRemoved(volumeId); - long volumeStoragePoolId = volumeVO.getPoolId(); + long volumeStoragePoolId = (volumeVO.getPoolId() != null ? volumeVO.getPoolId() : volumeVO.getLastPoolId()); if (SnapshotOperation.REVERT.equals(op)) { boolean baseVolumeExists = volumeVO.getRemoved() == null; diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java index d0184359c8b..b6029c27148 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java @@ -126,7 +126,7 @@ public class PrimaryDataStoreImpl implements PrimaryDataStore { @Override public List getVolumes() { - List volumes = volumeDao.findByPoolId(getId()); + List volumes = volumeDao.findNonDestroyedVolumesByPoolId(getId()); List volumeInfos = new ArrayList(); for (VolumeVO volume : volumes) { volumeInfos.add(VolumeObject.getVolumeObject(this, volume)); diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java index 78b3088415a..7132d61ed3b 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java @@ -387,6 +387,7 @@ public class VolumeServiceImpl implements VolumeService { logger.info("Expunge volume with no data store specified"); if (canVolumeBeRemoved(volume.getId())) { logger.info("Volume {} is not referred anywhere, remove it from volumes table", volume); + snapshotMgr.deletePoliciesForVolume(volume.getId()); volDao.remove(volume.getId()); } future.complete(result); @@ -422,6 +423,7 @@ public class VolumeServiceImpl implements VolumeService { } VMTemplateVO template = templateDao.findById(vol.getTemplateId()); if (template != null && !template.isDeployAsIs()) { + snapshotMgr.deletePoliciesForVolume(vol.getId()); volDao.remove(vol.getId()); future.complete(result); return future; @@ -493,6 +495,7 @@ public class VolumeServiceImpl implements VolumeService { if (canVolumeBeRemoved(vo.getId())) { logger.info("Volume {} is not referred anywhere, remove it from volumes table", vo); + snapshotMgr.deletePoliciesForVolume(vo.getId()); volDao.remove(vo.getId()); } @@ -1656,7 +1659,6 @@ public class VolumeServiceImpl implements VolumeService { // mark volume entry in volumes table as destroy state VolumeInfo vol = volFactory.getVolume(volumeId); vol.stateTransit(Volume.Event.DestroyRequested); - snapshotMgr.deletePoliciesForVolume(volumeId); annotationDao.removeByEntityType(AnnotationService.EntityType.VOLUME.name(), vol.getUuid()); vol.stateTransit(Volume.Event.OperationSucceeded); diff --git a/extensions/HyperV/hyperv.py b/extensions/HyperV/hyperv.py index c9b1d4da77e..8e2858d3cae 100755 --- a/extensions/HyperV/hyperv.py +++ b/extensions/HyperV/hyperv.py @@ -210,6 +210,29 @@ class HyperVManager: power_state = "poweroff" succeed({"status": "success", "power_state": power_state}) + def statuses(self): + command = 'Get-VM | Select-Object Name, State | ConvertTo-Json' + output = self.run_ps(command) + if not output or output.strip() in ("", "null"): + vms = [] + else: + try: + vms = json.loads(output) + except json.JSONDecodeError: + fail("Failed to parse VM status output: " + output) + power_state = {} + if isinstance(vms, dict): + vms = [vms] + for vm in vms: + state = vm["State"].strip().lower() + if state == "running": + power_state[vm["Name"]] = "poweron" + elif state == "off": + power_state[vm["Name"]] = "poweroff" + else: + power_state[vm["Name"]] = "unknown" + succeed({"status": "success", "power_state": power_state}) + def delete(self): try: self.run_ps_int(f'Remove-VM -Name "{self.data["vmname"]}" -Force') @@ -286,6 +309,7 @@ def main(): "reboot": manager.reboot, "delete": manager.delete, "status": manager.status, + "statuses": manager.statuses, "getconsole": manager.get_console, "suspend": manager.suspend, "resume": manager.resume, diff --git a/extensions/Proxmox/proxmox.sh b/extensions/Proxmox/proxmox.sh index fc27f2f3075..459b744f44d 100755 --- a/extensions/Proxmox/proxmox.sh +++ b/extensions/Proxmox/proxmox.sh @@ -64,7 +64,7 @@ parse_json() { token="${host_token:-$extension_token}" secret="${host_secret:-$extension_secret}" - check_required_fields vm_internal_name url user token secret node + check_required_fields url user token secret node } urlencode() { @@ -206,6 +206,10 @@ prepare() { create() { if [[ -z "$vm_name" ]]; then + if [[ -z "$vm_internal_name" ]]; then + echo '{"error":"Missing required fields: vm_internal_name"}' + exit 1 + fi vm_name="$vm_internal_name" fi validate_name "VM" "$vm_name" @@ -331,71 +335,102 @@ get_node_host() { echo "$host" } - get_console() { - check_required_fields node vmid +get_console() { + check_required_fields node vmid - local api_resp port ticket - if ! api_resp="$(call_proxmox_api POST "/nodes/${node}/qemu/${vmid}/vncproxy")"; then - echo "$api_resp" | jq -c '{status:"error", error:(.errors.curl // (.errors|tostring))}' - exit 1 - fi + local api_resp port ticket + if ! api_resp="$(call_proxmox_api POST "/nodes/${node}/qemu/${vmid}/vncproxy")"; then + echo "$api_resp" | jq -c '{status:"error", error:(.errors.curl // (.errors|tostring))}' + exit 1 + fi - port="$(echo "$api_resp" | jq -re '.data.port // empty' 2>/dev/null || true)" - ticket="$(echo "$api_resp" | jq -re '.data.ticket // empty' 2>/dev/null || true)" + port="$(echo "$api_resp" | jq -re '.data.port // empty' 2>/dev/null || true)" + ticket="$(echo "$api_resp" | jq -re '.data.ticket // empty' 2>/dev/null || true)" - if [[ -z "$port" || -z "$ticket" ]]; then - jq -n --arg raw "$api_resp" \ - '{status:"error", error:"Proxmox response missing port/ticket", upstream:$raw}' - exit 1 - fi + if [[ -z "$port" || -z "$ticket" ]]; then + jq -n --arg raw "$api_resp" \ + '{status:"error", error:"Proxmox response missing port/ticket", upstream:$raw}' + exit 1 + fi - # Derive host from node’s network info - local host - host="$(get_node_host)" - if [[ -z "$host" ]]; then - jq -n --arg msg "Could not determine host IP for node $node" \ - '{status:"error", error:$msg}' - exit 1 - fi + # Derive host from node’s network info + local host + host="$(get_node_host)" + if [[ -z "$host" ]]; then + jq -n --arg msg "Could not determine host IP for node $node" \ + '{status:"error", error:$msg}' + exit 1 + fi - jq -n \ - --arg host "$host" \ - --arg port "$port" \ - --arg password "$ticket" \ - --argjson passwordonetimeuseonly true \ - '{ - status: "success", - message: "Console retrieved", - console: { - host: $host, - port: $port, - password: $password, - passwordonetimeuseonly: $passwordonetimeuseonly, - protocol: "vnc" - } - }' - } + jq -n \ + --arg host "$host" \ + --arg port "$port" \ + --arg password "$ticket" \ + --argjson passwordonetimeuseonly true \ + '{ + status: "success", + message: "Console retrieved", + console: { + host: $host, + port: $port, + password: $password, + passwordonetimeuseonly: $passwordonetimeuseonly, + protocol: "vnc" + } + }' +} + +statuses() { + local response + response=$(call_proxmox_api GET "/nodes/${node}/qemu") + + if [[ -z "$response" ]]; then + echo '{"status":"error","message":"empty response from Proxmox API"}' + return 1 + fi + + if ! echo "$response" | jq empty >/dev/null 2>&1; then + echo '{"status":"error","message":"invalid JSON response from Proxmox API"}' + return 1 + fi + + echo "$response" | jq -c ' + def map_state(s): + if s=="running" then "poweron" + elif s=="stopped" then "poweroff" + else "unknown" end; + + { + status: "success", + power_state: ( + .data + | map(select(.template != 1)) + | map({ ( (.name // (.vmid|tostring)) ): map_state(.status) }) + | add // {} + ) + }' +} list_snapshots() { snapshot_response=$(call_proxmox_api GET "/nodes/${node}/qemu/${vmid}/snapshot") echo "$snapshot_response" | jq ' - def to_date: - if . == "-" then "-" - elif . == null then "-" - else (. | tonumber | strftime("%Y-%m-%d %H:%M:%S")) - end; + def to_date: + if . == "-" then "-" + elif . == null then "-" + else (. | tonumber | strftime("%Y-%m-%d %H:%M:%S")) + end; - { - status: "success", - printmessage: "true", - message: [.data[] | { - name: .name, - snaptime: ((.snaptime // "-") | to_date), - description: .description, - parent: (.parent // "-"), - vmstate: (.vmstate // "-") - }] - } + { + status: "success", + printmessage: "true", + message: [.data[] | { + name: .name, + snaptime: ((.snaptime // "-") | to_date), + description: .description, + parent: (.parent // "-"), + vmstate: (.vmstate // "-") + }] + } ' } @@ -463,9 +498,9 @@ parse_json "$parameters" || exit 1 cleanup_vm=0 cleanup() { - if (( cleanup_vm == 1 )); then - execute_and_wait DELETE "/nodes/${node}/qemu/${vmid}" - fi + if (( cleanup_vm == 1 )); then + execute_and_wait DELETE "/nodes/${node}/qemu/${vmid}" + fi } trap cleanup EXIT @@ -492,6 +527,9 @@ case $action in status) status ;; + statuses) + statuses + ;; getconsole) get_console ;; diff --git a/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java b/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java index 0ea910fcb6d..c19ec751153 100644 --- a/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java +++ b/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java @@ -267,10 +267,18 @@ public class ConfigKey { static ConfigDepotImpl s_depot = null; - static public void init(ConfigDepotImpl depot) { + private String _defaultValueIfEmpty = null; + + public static void init(ConfigDepotImpl depot) { s_depot = depot; } + public ConfigKey(Class type, String name, String category, String defaultValue, String description, boolean isDynamic, Scope scope, T multiplier, + String displayText, String parent, Ternary group, Pair subGroup, Kind kind, String options, String defaultValueIfEmpty) { + this(type, name, category, defaultValue, description, isDynamic, scope, multiplier, displayText, parent, group, subGroup, kind, options); + this._defaultValueIfEmpty = defaultValueIfEmpty; + } + public ConfigKey(String category, Class type, String name, String defaultValue, String description, boolean isDynamic, Scope scope) { this(type, name, category, defaultValue, description, isDynamic, scope, null); } @@ -380,7 +388,19 @@ public class ConfigKey { public T value() { if (_value == null || isDynamic()) { String value = s_depot != null ? s_depot.getConfigStringValue(_name, Scope.Global, null) : null; - _value = valueOf((value == null) ? defaultValue() : value); + + String effective; + if (value != null) { + if (value.isEmpty() && _defaultValueIfEmpty != null) { + effective = _defaultValueIfEmpty; + } else { + effective = value; + } + } else { + effective = _defaultValueIfEmpty != null ? _defaultValueIfEmpty : defaultValue(); + } + + _value = valueOf(effective); } return _value; } @@ -409,6 +429,10 @@ public class ConfigKey { return valueInGlobalOrAvailableParentScope(scope, id); } logger.trace("Scope({}) value for config ({}): {}", scope, _name, _value); + + if (value.isEmpty() && _defaultValueIfEmpty != null) { + return valueOf(_defaultValueIfEmpty); + } return valueOf(value); } diff --git a/framework/db/pom.xml b/framework/db/pom.xml index 04d0fcc7e89..ced0f64c663 100644 --- a/framework/db/pom.xml +++ b/framework/db/pom.xml @@ -53,8 +53,8 @@ commons-pool2 - mysql - mysql-connector-java + com.mysql + mysql-connector-j org.apache.cloudstack diff --git a/framework/db/src/main/java/com/cloud/utils/db/Filter.java b/framework/db/src/main/java/com/cloud/utils/db/Filter.java index fb8c9ee37fc..90e42952a99 100644 --- a/framework/db/src/main/java/com/cloud/utils/db/Filter.java +++ b/framework/db/src/main/java/com/cloud/utils/db/Filter.java @@ -57,7 +57,8 @@ public class Filter { } public Filter(long limit) { - _orderBy = " ORDER BY RAND() LIMIT " + limit; + _orderBy = " ORDER BY RAND()"; + _limit = limit; } public Filter(Long offset, Long limit) { diff --git a/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java b/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java index 560cb4494ee..87d9f11d20b 100644 --- a/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java +++ b/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java @@ -89,6 +89,7 @@ import net.sf.cglib.proxy.NoOp; import net.sf.ehcache.Cache; import net.sf.ehcache.CacheManager; import net.sf.ehcache.Element; +import org.springframework.util.ClassUtils; /** * GenericDaoBase is a simple way to implement DAOs. It DOES NOT @@ -1171,6 +1172,8 @@ public abstract class GenericDaoBase extends Compone if (filter.getLimit() != null) { sql.append(", ").append(filter.getLimit()); } + } else if (filter.getLimit() != null) { + sql.append(" LIMIT ").append(filter.getLimit()); } } } @@ -1332,7 +1335,7 @@ public abstract class GenericDaoBase extends Compone Filter filter = null; final long batchSizeFinal = ObjectUtils.defaultIfNull(batchSize, 0L); if (batchSizeFinal > 0) { - filter = new Filter(batchSizeFinal); + filter = new Filter(null, batchSizeFinal); } int expunged = 0; int currentExpunged = 0; @@ -2058,16 +2061,22 @@ public abstract class GenericDaoBase extends Compone @DB() protected void setField(final Object entity, final ResultSet rs, ResultSetMetaData meta, final int index) throws SQLException { - Attribute attr = _allColumns.get(new Pair(meta.getTableName(index), meta.getColumnName(index))); + String tableName = meta.getTableName(index); + String columnName = meta.getColumnName(index); + Attribute attr = _allColumns.get(new Pair<>(tableName, columnName)); if (attr == null) { // work around for mysql bug to return original table name instead of view name in db view case Table tbl = entity.getClass().getSuperclass().getAnnotation(Table.class); if (tbl != null) { - attr = _allColumns.get(new Pair(tbl.name(), meta.getColumnLabel(index))); + attr = _allColumns.get(new Pair<>(tbl.name(), meta.getColumnLabel(index))); } } - assert (attr != null) : "How come I can't find " + meta.getCatalogName(index) + "." + meta.getColumnName(index); - setField(entity, attr.field, rs, index); + if(attr == null) { + logger.warn(String.format("Failed to find attribute in the entity %s to map column %s.%s (%s)", + ClassUtils.getUserClass(entity).getSimpleName(), tableName, columnName)); + } else { + setField(entity, attr.field, rs, index); + } } @Override diff --git a/packaging/el8/cloud.spec b/packaging/el8/cloud.spec index c5079a8aa94..705959336f1 100644 --- a/packaging/el8/cloud.spec +++ b/packaging/el8/cloud.spec @@ -76,7 +76,7 @@ Requires: sudo Requires: /sbin/service Requires: /sbin/chkconfig Requires: /usr/bin/ssh-keygen -Requires: (genisoimage or mkisofs) +Requires: (genisoimage or mkisofs or xorrisofs) Requires: ipmitool Requires: %{name}-common = %{_ver} Requires: (iptables-services or iptables) diff --git a/plugins/api/discovery/src/main/java/org/apache/cloudstack/api/response/ApiDiscoveryResponse.java b/plugins/api/discovery/src/main/java/org/apache/cloudstack/api/response/ApiDiscoveryResponse.java index 4b243f2e8a1..90b3b89d3fb 100644 --- a/plugins/api/discovery/src/main/java/org/apache/cloudstack/api/response/ApiDiscoveryResponse.java +++ b/plugins/api/discovery/src/main/java/org/apache/cloudstack/api/response/ApiDiscoveryResponse.java @@ -59,6 +59,10 @@ public class ApiDiscoveryResponse extends BaseResponse { @Param(description = "Response field type") private String type; + @SerializedName(ApiConstants.HTTP_REQUEST_TYPE) + @Param(description = "Preferred HTTP request type for the API", since = "4.23.0") + private String httpRequestType; + public ApiDiscoveryResponse() { params = new HashSet(); apiResponse = new HashSet(); @@ -74,6 +78,7 @@ public class ApiDiscoveryResponse extends BaseResponse { this.params = new HashSet<>(another.getParams()); this.apiResponse = new HashSet<>(another.getApiResponse()); this.type = another.getType(); + this.httpRequestType = another.getHttpRequestType(); this.setObjectName(another.getObjectName()); } @@ -140,4 +145,12 @@ public class ApiDiscoveryResponse extends BaseResponse { public String getType() { return type; } + + public String getHttpRequestType() { + return httpRequestType; + } + + public void setHttpRequestType(String httpRequestType) { + this.httpRequestType = httpRequestType; + } } diff --git a/plugins/api/discovery/src/main/java/org/apache/cloudstack/discovery/ApiDiscoveryServiceImpl.java b/plugins/api/discovery/src/main/java/org/apache/cloudstack/discovery/ApiDiscoveryServiceImpl.java index 452b95cf2c0..d6d235162ef 100644 --- a/plugins/api/discovery/src/main/java/org/apache/cloudstack/discovery/ApiDiscoveryServiceImpl.java +++ b/plugins/api/discovery/src/main/java/org/apache/cloudstack/discovery/ApiDiscoveryServiceImpl.java @@ -50,6 +50,7 @@ import org.apache.commons.lang3.StringUtils; import org.reflections.ReflectionUtils; import org.springframework.stereotype.Component; +import com.cloud.api.ApiServlet; import com.cloud.exception.PermissionDeniedException; import com.cloud.serializer.Param; import com.cloud.user.Account; @@ -189,7 +190,7 @@ public class ApiDiscoveryServiceImpl extends ComponentLifecycleBase implements A return responseResponse; } - private ApiDiscoveryResponse getCmdRequestMap(Class cmdClass, APICommand apiCmdAnnotation) { + protected ApiDiscoveryResponse getCmdRequestMap(Class cmdClass, APICommand apiCmdAnnotation) { String apiName = apiCmdAnnotation.name(); ApiDiscoveryResponse response = new ApiDiscoveryResponse(); response.setName(apiName); @@ -197,6 +198,12 @@ public class ApiDiscoveryServiceImpl extends ComponentLifecycleBase implements A if (!apiCmdAnnotation.since().isEmpty()) { response.setSince(apiCmdAnnotation.since()); } + String httpRequestType = apiCmdAnnotation.httpMethod(); + if (StringUtils.isBlank(httpRequestType)) { + httpRequestType = ApiServlet.GET_REQUEST_COMMANDS.matcher(apiName.toLowerCase()).matches() ? + "GET" : "POST"; + } + response.setHttpRequestType(httpRequestType); Set fields = ReflectUtil.getAllFieldsForClass(cmdClass, new Class[] {BaseCmd.class, BaseAsyncCmd.class, BaseAsyncCreateCmd.class}); diff --git a/plugins/api/discovery/src/test/java/org/apache/cloudstack/discovery/ApiDiscoveryServiceImplTest.java b/plugins/api/discovery/src/test/java/org/apache/cloudstack/discovery/ApiDiscoveryServiceImplTest.java new file mode 100644 index 00000000000..e69b9523d44 --- /dev/null +++ b/plugins/api/discovery/src/test/java/org/apache/cloudstack/discovery/ApiDiscoveryServiceImplTest.java @@ -0,0 +1,123 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.discovery; + +import static org.mockito.ArgumentMatchers.any; + +import java.lang.reflect.Field; +import java.util.Set; + +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.command.admin.account.CreateAccountCmd; +import org.apache.cloudstack.api.command.admin.user.GetUserCmd; +import org.apache.cloudstack.api.command.user.discovery.ListApisCmd; +import org.apache.cloudstack.api.response.ApiDiscoveryResponse; +import org.apache.cloudstack.api.response.ApiParameterResponse; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; + +import com.cloud.utils.ReflectUtil; + +@RunWith(MockitoJUnitRunner.class) +public class ApiDiscoveryServiceImplTest { + + @Mock + APICommand apiCommandMock; + + @Spy + @InjectMocks + ApiDiscoveryServiceImpl discoveryServiceSpy; + + @Before + public void setUp() { + Mockito.when(apiCommandMock.name()).thenReturn("listApis"); + Mockito.when(apiCommandMock.since()).thenReturn(""); + } + + @Test + public void getCmdRequestMapReturnsResponseWithCorrectApiNameAndDescription() { + Mockito.when(apiCommandMock.description()).thenReturn("Lists all APIs"); + ApiDiscoveryResponse response = discoveryServiceSpy.getCmdRequestMap(ListApisCmd.class, apiCommandMock); + Assert.assertEquals("listApis", response.getName()); + Assert.assertEquals("Lists all APIs", response.getDescription()); + } + + @Test + public void getCmdRequestMapSetsHttpRequestTypeToGetWhenApiNameMatchesGetPattern() { + Mockito.when(apiCommandMock.name()).thenReturn("getUser"); + Mockito.when(apiCommandMock.httpMethod()).thenReturn(""); + ApiDiscoveryResponse response = discoveryServiceSpy.getCmdRequestMap(GetUserCmd.class, apiCommandMock); + Assert.assertEquals("GET", response.getHttpRequestType()); + } + + @Test + public void getCmdRequestMapSetsHttpRequestTypeToPostWhenApiNameDoesNotMatchGetPattern() { + Mockito.when(apiCommandMock.name()).thenReturn("createAccount"); + Mockito.when(apiCommandMock.httpMethod()).thenReturn(""); + ApiDiscoveryResponse response = discoveryServiceSpy.getCmdRequestMap(CreateAccountCmd.class, apiCommandMock); + Assert.assertEquals("POST", response.getHttpRequestType()); + } + + @Test + public void getCmdRequestMapSetsAsyncToTrueForAsyncCommand() { + Mockito.when(apiCommandMock.name()).thenReturn("asyncApi"); + ApiDiscoveryResponse response = discoveryServiceSpy.getCmdRequestMap(BaseAsyncCmd.class, apiCommandMock); + Assert.assertTrue(response.getAsync()); + } + + @Test + public void getCmdRequestMapDoesNotAddParamsWithoutParameterAnnotation() { + ApiDiscoveryResponse response = discoveryServiceSpy.getCmdRequestMap(BaseCmd.class, apiCommandMock); + Assert.assertFalse(response.getParams().isEmpty()); + Assert.assertEquals(1, response.getParams().size()); + } + + @Test + public void getCmdRequestMapAddsParamsWithExposedAndIncludedInApiDocAnnotations() { + Field fieldMock = Mockito.mock(Field.class); + Parameter parameterMock = Mockito.mock(Parameter.class); + Mockito.when(parameterMock.expose()).thenReturn(true); + Mockito.when(parameterMock.includeInApiDoc()).thenReturn(true); + Mockito.when(parameterMock.name()).thenReturn("paramName"); + Mockito.when(parameterMock.since()).thenReturn(""); + Mockito.when(parameterMock.entityType()).thenReturn(new Class[]{Object.class}); + Mockito.when(parameterMock.description()).thenReturn("paramDescription"); + Mockito.when(parameterMock.type()).thenReturn(BaseCmd.CommandType.STRING); + Mockito.when(fieldMock.getAnnotation(Parameter.class)).thenReturn(parameterMock); + try (MockedStatic reflectUtilMockedStatic = Mockito.mockStatic(ReflectUtil.class)) { + reflectUtilMockedStatic.when(() -> ReflectUtil.getAllFieldsForClass(any(Class.class), any(Class[].class))) + .thenReturn(Set.of(fieldMock)); + ApiDiscoveryResponse response = discoveryServiceSpy.getCmdRequestMap(ListApisCmd.class, apiCommandMock); + Set params = response.getParams(); + Assert.assertEquals(1, params.size()); + ApiParameterResponse paramResponse = params.iterator().next(); + Assert.assertEquals("paramName", ReflectionTestUtils.getField(paramResponse, "name")); + } + } +} diff --git a/plugins/api/vmware-sioc/src/main/java/org/apache/cloudstack/sioc/SiocManagerImpl.java b/plugins/api/vmware-sioc/src/main/java/org/apache/cloudstack/sioc/SiocManagerImpl.java index e93b8df39e9..b01af35725f 100644 --- a/plugins/api/vmware-sioc/src/main/java/org/apache/cloudstack/sioc/SiocManagerImpl.java +++ b/plugins/api/vmware-sioc/src/main/java/org/apache/cloudstack/sioc/SiocManagerImpl.java @@ -123,7 +123,7 @@ public class SiocManagerImpl implements SiocManager { int limitIopsTotal = 0; - List volumes = volumeDao.findByPoolId(storagePoolId, null); + List volumes = volumeDao.findNonDestroyedVolumesByPoolId(storagePoolId, null); if (volumes != null && volumes.size() > 0) { Set instanceIds = new HashSet<>(); diff --git a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java index fe3633dab16..8a111f92868 100644 --- a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java +++ b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java @@ -364,7 +364,9 @@ public class VeeamClient { * that is used to wait for the restore to complete before throwing a {@link CloudRuntimeException}. */ protected void checkIfRestoreSessionFinished(String type, String path) throws IOException { - for (int j = 0; j < restoreTimeout; j++) { + long startTime = System.currentTimeMillis(); + long timeoutMs = restoreTimeout * 1000L; + while (System.currentTimeMillis() - startTime < timeoutMs) { HttpResponse relatedResponse = get(path); RestoreSession session = parseRestoreSessionResponse(relatedResponse); if (session.getResult().equals("Success")) { @@ -378,7 +380,8 @@ public class VeeamClient { getRestoreVmErrorDescription(StringUtils.substringAfterLast(sessionUid, ":")))); throw new CloudRuntimeException(String.format("Restore job [%s] failed.", sessionUid)); } - logger.debug(String.format("Waiting %s seconds, out of a total of %s seconds, for the restore backup process to finish.", j, restoreTimeout)); + logger.debug("Waiting {} seconds, out of a total of {} seconds, for the restore backup process to finish.", + (System.currentTimeMillis() - startTime) / 1000, restoreTimeout); try { Thread.sleep(1000); diff --git a/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java b/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java index 0c70c75939e..333c3e16053 100644 --- a/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java +++ b/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java @@ -25,7 +25,6 @@ import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor; import static com.github.tomakehurst.wiremock.client.WireMock.urlMatching; import static com.github.tomakehurst.wiremock.client.WireMock.verify; import static org.junit.Assert.fail; -import static org.mockito.Mockito.times; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -157,7 +156,7 @@ public class VeeamClientTest { @Test public void checkIfRestoreSessionFinishedTestTimeoutException() throws IOException { try { - ReflectionTestUtils.setField(mockClient, "restoreTimeout", 10); + ReflectionTestUtils.setField(mockClient, "restoreTimeout", 2); RestoreSession restoreSession = Mockito.mock(RestoreSession.class); HttpResponse httpResponse = Mockito.mock(HttpResponse.class); Mockito.when(mockClient.get(Mockito.anyString())).thenReturn(httpResponse); @@ -169,7 +168,7 @@ public class VeeamClientTest { } catch (Exception e) { Assert.assertEquals("Related job type: RestoreTest was not successful", e.getMessage()); } - Mockito.verify(mockClient, times(10)).get(Mockito.anyString()); + Mockito.verify(mockClient, Mockito.atLeastOnce()).get(Mockito.anyString()); } @Test diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaBalanceCmd.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaBalanceCmd.java index cf39f802d34..0cec0df6618 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaBalanceCmd.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaBalanceCmd.java @@ -35,7 +35,8 @@ import org.apache.cloudstack.api.response.QuotaResponseBuilder; import org.apache.cloudstack.quota.vo.QuotaBalanceVO; import org.apache.cloudstack.api.response.QuotaStatementItemResponse; -@APICommand(name = "quotaBalance", responseObject = QuotaStatementItemResponse.class, description = "Create a quota balance statement", since = "4.7.0", requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) +@APICommand(name = "quotaBalance", responseObject = QuotaStatementItemResponse.class, description = "Create a quota balance statement", since = "4.7.0", requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, + httpMethod = "GET") public class QuotaBalanceCmd extends BaseCmd { diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaEnabledCmd.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaEnabledCmd.java index 4035a5205e6..af1d146ea9d 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaEnabledCmd.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaEnabledCmd.java @@ -26,7 +26,8 @@ import org.apache.cloudstack.quota.QuotaService; import javax.inject.Inject; -@APICommand(name = "quotaIsEnabled", responseObject = QuotaEnabledResponse.class, description = "Return true if the plugin is enabled", since = "4.7.0", requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) +@APICommand(name = "quotaIsEnabled", responseObject = QuotaEnabledResponse.class, description = "Return true if the plugin is enabled", since = "4.7.0", requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, + httpMethod = "GET") public class QuotaEnabledCmd extends BaseCmd { diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaStatementCmd.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaStatementCmd.java index 18f9bc48a6e..d3bd3868ed1 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaStatementCmd.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaStatementCmd.java @@ -35,7 +35,8 @@ import org.apache.cloudstack.quota.vo.QuotaUsageVO; import com.cloud.user.Account; -@APICommand(name = "quotaStatement", responseObject = QuotaStatementItemResponse.class, description = "Create a quota statement", since = "4.7.0", requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) +@APICommand(name = "quotaStatement", responseObject = QuotaStatementItemResponse.class, description = "Create a quota statement", since = "4.7.0", requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, + httpMethod = "GET") public class QuotaStatementCmd extends BaseCmd { diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaSummaryCmd.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaSummaryCmd.java index 42a598042b3..87322b01f4d 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaSummaryCmd.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaSummaryCmd.java @@ -33,7 +33,8 @@ import java.util.List; import javax.inject.Inject; -@APICommand(name = "quotaSummary", responseObject = QuotaSummaryResponse.class, description = "Lists balance and quota usage for all Accounts", since = "4.7.0", requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) +@APICommand(name = "quotaSummary", responseObject = QuotaSummaryResponse.class, description = "Lists balance and quota usage for all Accounts", since = "4.7.0", requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, + httpMethod = "GET") public class QuotaSummaryCmd extends BaseListCmd { @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, required = false, description = "Optional, Account Id for which statement needs to be generated") diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaTariffListCmd.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaTariffListCmd.java index d054d545931..e0bab07501b 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaTariffListCmd.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaTariffListCmd.java @@ -38,7 +38,8 @@ import java.util.ArrayList; import java.util.Date; import java.util.List; -@APICommand(name = "quotaTariffList", responseObject = QuotaTariffResponse.class, description = "Lists all quota tariff plans", since = "4.7.0", requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) +@APICommand(name = "quotaTariffList", responseObject = QuotaTariffResponse.class, description = "Lists all quota tariff plans", since = "4.7.0", requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, + httpMethod = "GET") public class QuotaTariffListCmd extends BaseListCmd { @Inject diff --git a/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalDnsmasqResource.java b/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalDnsmasqResource.java index 51acfe93d39..8e7efedfca3 100644 --- a/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalDnsmasqResource.java +++ b/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalDnsmasqResource.java @@ -46,10 +46,10 @@ public class BaremetalDnsmasqResource extends BaremetalDhcpResourceBase { com.trilead.ssh2.Connection sshConnection = null; try { super.configure(name, params); - logger.debug(String.format("Trying to connect to DHCP server(IP=%1$s, username=%2$s, password=%3$s)", _ip, _username, _password)); + logger.debug(String.format("Trying to connect to DHCP server(IP=%1$s, username=%2$s", _ip, _username)); sshConnection = SSHCmdHelper.acquireAuthorizedConnection(_ip, _username, _password); if (sshConnection == null) { - throw new ConfigurationException(String.format("Cannot connect to DHCP server(IP=%1$s, username=%2$s, password=%3$s", _ip, _username, _password)); + throw new ConfigurationException(String.format("Cannot connect to DHCP server(IP=%1$s, username=%2$s", _ip, _username)); } if (!SSHCmdHelper.sshExecuteCmd(sshConnection, "[ -f '/usr/sbin/dnsmasq' ]")) { diff --git a/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalKickStartPxeResource.java b/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalKickStartPxeResource.java index 3775f4effc1..88c4dea96b3 100644 --- a/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalKickStartPxeResource.java +++ b/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalKickStartPxeResource.java @@ -130,8 +130,8 @@ public class BaremetalKickStartPxeResource extends BaremetalPxeResourceBase { sshConnection.connect(null, 60000, 60000); if (!sshConnection.authenticateWithPassword(_username, _password)) { - logger.debug("SSH Failed to authenticate"); - throw new ConfigurationException(String.format("Cannot connect to PING PXE server(IP=%1$s, username=%2$s, password=%3$s", _ip, _username, _password)); + logger.debug("SSH Failed to authenticate with user {} credentials", _username); + throw new ConfigurationException(String.format("Cannot connect to PING PXE server(IP=%1$s, username=%2$s", _ip, _username)); } String script = String.format("python /usr/bin/baremetal_user_data.py '%s'", arg); @@ -167,7 +167,7 @@ public class BaremetalKickStartPxeResource extends BaremetalPxeResourceBase { sshConnection.connect(null, 60000, 60000); if (!sshConnection.authenticateWithPassword(_username, _password)) { logger.debug("SSH Failed to authenticate"); - throw new ConfigurationException(String.format("Cannot connect to PING PXE server(IP=%1$s, username=%2$s, password=%3$s", _ip, _username, _password)); + throw new ConfigurationException(String.format("Cannot connect to PING PXE server(IP=%1$s, username=%2$s", _ip, _username)); } String copyTo = String.format("%s/%s", _tftpDir, cmd.getTemplateUuid()); diff --git a/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalPingPxeResource.java b/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalPingPxeResource.java index 96b2dbfeb93..a54cd4a1a11 100644 --- a/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalPingPxeResource.java +++ b/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalPingPxeResource.java @@ -101,7 +101,7 @@ public class BaremetalPingPxeResource extends BaremetalPxeResourceBase { sshConnection.connect(null, 60000, 60000); if (!sshConnection.authenticateWithPassword(_username, _password)) { logger.debug("SSH Failed to authenticate"); - throw new ConfigurationException(String.format("Cannot connect to PING PXE server(IP=%1$s, username=%2$s, password=%3$s", _ip, _username, "******")); + throw new ConfigurationException(String.format("Cannot connect to PING PXE server(IP=%1$s, username=%2$s, password=******", _ip, _username)); } String cmd = String.format("[ -f /%1$s/pxelinux.0 ] && [ -f /%2$s/kernel ] && [ -f /%3$s/initrd.gz ] ", _tftpDir, _tftpDir, _tftpDir); @@ -150,8 +150,8 @@ public class BaremetalPingPxeResource extends BaremetalPxeResourceBase { try { sshConnection.connect(null, 60000, 60000); if (!sshConnection.authenticateWithPassword(_username, _password)) { - logger.debug("SSH Failed to authenticate"); - throw new ConfigurationException(String.format("Cannot connect to PING PXE server(IP=%1$s, username=%2$s, password=%3$s", _ip, _username, _password)); + logger.debug("SSH Failed to authenticate with user {} credentials", _username); + throw new ConfigurationException(String.format("Cannot connect to PING PXE server(IP=%1$s, username=%2$s", _ip, _username)); } String script = @@ -179,7 +179,7 @@ public class BaremetalPingPxeResource extends BaremetalPxeResourceBase { sshConnection.connect(null, 60000, 60000); if (!sshConnection.authenticateWithPassword(_username, _password)) { logger.debug("SSH Failed to authenticate"); - throw new ConfigurationException(String.format("Cannot connect to PING PXE server(IP=%1$s, username=%2$s, password=%3$s", _ip, _username, _password)); + throw new ConfigurationException(String.format("Cannot connect to PING PXE server(IP=%1$s, username=%2$s", _ip, _username)); } String script = @@ -237,7 +237,7 @@ public class BaremetalPingPxeResource extends BaremetalPxeResourceBase { sshConnection.connect(null, 60000, 60000); if (!sshConnection.authenticateWithPassword(_username, _password)) { logger.debug("SSH Failed to authenticate"); - throw new ConfigurationException(String.format("Cannot connect to PING PXE server(IP=%1$s, username=%2$s, password=%3$s", _ip, _username, _password)); + throw new ConfigurationException(String.format("Cannot connect to PING PXE server(IP=%1$s, username=%2$s", _ip, _username)); } String script = String.format("python /usr/bin/baremetal_user_data.py '%s'", arg); diff --git a/plugins/hypervisors/external/src/main/java/org/apache/cloudstack/hypervisor/external/provisioner/ExternalPathPayloadProvisioner.java b/plugins/hypervisors/external/src/main/java/org/apache/cloudstack/hypervisor/external/provisioner/ExternalPathPayloadProvisioner.java index 92205b13c6f..fa3f4de5026 100644 --- a/plugins/hypervisors/external/src/main/java/org/apache/cloudstack/hypervisor/external/provisioner/ExternalPathPayloadProvisioner.java +++ b/plugins/hypervisors/external/src/main/java/org/apache/cloudstack/hypervisor/external/provisioner/ExternalPathPayloadProvisioner.java @@ -71,6 +71,7 @@ import com.cloud.agent.api.StartCommand; import com.cloud.agent.api.StopAnswer; import com.cloud.agent.api.StopCommand; import com.cloud.agent.api.to.VirtualMachineTO; +import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.ExternalProvisioner; @@ -128,7 +129,7 @@ public class ExternalPathPayloadProvisioner extends ManagerBase implements Exter private ExecutorService payloadCleanupExecutor; private ScheduledExecutorService payloadCleanupScheduler; private static final List TRIVIAL_ACTIONS = Arrays.asList( - "status" + "status", "statuses" ); @Override @@ -456,7 +457,7 @@ public class ExternalPathPayloadProvisioner extends ManagerBase implements Exter @Override public Map getHostVmStateReport(long hostId, String extensionName, String extensionRelativePath) { - final Map vmStates = new HashMap<>(); + Map vmStates = new HashMap<>(); String extensionPath = getExtensionCheckedPath(extensionName, extensionRelativePath); if (StringUtils.isEmpty(extensionPath)) { return vmStates; @@ -466,14 +467,20 @@ public class ExternalPathPayloadProvisioner extends ManagerBase implements Exter logger.error("Host with ID: {} not found", hostId); return vmStates; } + Map> accessDetails = + extensionsManager.getExternalAccessDetails(host, null); + vmStates = getVmPowerStates(host, accessDetails, extensionName, extensionPath); + if (vmStates != null) { + logger.debug("Found {} VMs on the host {}", vmStates.size(), host); + return vmStates; + } + vmStates = new HashMap<>(); List allVms = _uservmDao.listByHostId(hostId); allVms.addAll(_uservmDao.listByLastHostId(hostId)); if (CollectionUtils.isEmpty(allVms)) { logger.debug("No VMs found for the {}", host); return vmStates; } - Map> accessDetails = - extensionsManager.getExternalAccessDetails(host, null); for (UserVmVO vm: allVms) { VirtualMachine.PowerState powerState = getVmPowerState(vm, accessDetails, extensionName, extensionPath); vmStates.put(vm.getInstanceName(), new HostVmStateReportEntry(powerState, "host-" + hostId)); @@ -714,7 +721,7 @@ public class ExternalPathPayloadProvisioner extends ManagerBase implements Exter return getPowerStateFromString(response); } try { - JsonObject jsonObj = new JsonParser().parse(response).getAsJsonObject(); + JsonObject jsonObj = JsonParser.parseString(response).getAsJsonObject(); String powerState = jsonObj.has("power_state") ? jsonObj.get("power_state").getAsString() : null; return getPowerStateFromString(powerState); } catch (Exception e) { @@ -724,7 +731,7 @@ public class ExternalPathPayloadProvisioner extends ManagerBase implements Exter } } - private VirtualMachine.PowerState getVmPowerState(UserVmVO userVmVO, Map> accessDetails, + protected VirtualMachine.PowerState getVmPowerState(UserVmVO userVmVO, Map> accessDetails, String extensionName, String extensionPath) { VirtualMachineTO virtualMachineTO = getVirtualMachineTO(userVmVO); accessDetails.put(ApiConstants.VIRTUAL_MACHINE, virtualMachineTO.getExternalDetails()); @@ -740,6 +747,46 @@ public class ExternalPathPayloadProvisioner extends ManagerBase implements Exter } return parsePowerStateFromResponse(userVmVO, result.second()); } + + protected Map getVmPowerStates(Host host, + Map> accessDetails, String extensionName, String extensionPath) { + Map modifiedDetails = loadAccessDetails(accessDetails, null); + logger.debug("Trying to get VM power statuses from the external system for {}", host); + Pair result = getInstanceStatusesOnExternalSystem(extensionName, extensionPath, + host.getName(), modifiedDetails, AgentManager.Wait.value()); + if (!result.first()) { + logger.warn("Failure response received while trying to fetch the power statuses for {} : {}", + host, result.second()); + return null; + } + if (StringUtils.isBlank(result.second())) { + logger.warn("Empty response while trying to fetch VM power statuses for host: {}", host); + return null; + } + try { + JsonObject jsonObj = JsonParser.parseString(result.second()).getAsJsonObject(); + if (!jsonObj.has("status") || !"success".equalsIgnoreCase(jsonObj.get("status").getAsString())) { + logger.warn("Invalid status in response while trying to fetch VM power statuses for host: {}: {}", + host, result.second()); + return null; + } + if (!jsonObj.has("power_state") || !jsonObj.get("power_state").isJsonObject()) { + logger.warn("Missing or invalid power_state in response for host: {}: {}", host, result.second()); + return null; + } + JsonObject powerStates = jsonObj.getAsJsonObject("power_state"); + Map states = new HashMap<>(); + for (Map.Entry entry : powerStates.entrySet()) { + VirtualMachine.PowerState powerState = getPowerStateFromString(entry.getValue().getAsString()); + states.put(entry.getKey(), new HostVmStateReportEntry(powerState, "host-" + host.getId())); + } + return states; + } catch (Exception e) { + logger.warn("Failed to parse VM power statuses response for host: {}: {}", host, e.getMessage()); + return null; + } + } + public Pair prepareExternalProvisioningInternal(String extensionName, String filename, String vmUUID, Map accessDetails, int wait) { return executeExternalCommand(extensionName, "prepare", accessDetails, wait, @@ -783,6 +830,12 @@ public class ExternalPathPayloadProvisioner extends ManagerBase implements Exter String.format("Failed to get the instance power status %s on external system", vmUUID), filename); } + public Pair getInstanceStatusesOnExternalSystem(String extensionName, String filename, + String hostName, Map accessDetails, int wait) { + return executeExternalCommand(extensionName, "statuses", accessDetails, wait, + String.format("Failed to get the %s instances power status on external system", hostName), filename); + } + public Pair getInstanceConsoleOnExternalSystem(String extensionName, String filename, String vmUUID, Map accessDetails, int wait) { return executeExternalCommand(extensionName, "getconsole", accessDetails, wait, diff --git a/plugins/hypervisors/external/src/test/java/org/apache/cloudstack/hypervisor/external/provisioner/ExternalPathPayloadProvisionerTest.java b/plugins/hypervisors/external/src/test/java/org/apache/cloudstack/hypervisor/external/provisioner/ExternalPathPayloadProvisionerTest.java index d0a396f7a94..e8ab92c986e 100644 --- a/plugins/hypervisors/external/src/test/java/org/apache/cloudstack/hypervisor/external/provisioner/ExternalPathPayloadProvisionerTest.java +++ b/plugins/hypervisors/external/src/test/java/org/apache/cloudstack/hypervisor/external/provisioner/ExternalPathPayloadProvisionerTest.java @@ -79,6 +79,7 @@ import com.cloud.agent.api.StartCommand; import com.cloud.agent.api.StopAnswer; import com.cloud.agent.api.StopCommand; import com.cloud.agent.api.to.VirtualMachineTO; +import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor; @@ -761,6 +762,37 @@ public class ExternalPathPayloadProvisionerTest { assertNull(result); } + @Test + public void getVmPowerStatesReturnsValidStatesWhenResponseIsSuccessful() { + Host host = mock(Host.class); + when(host.getId()).thenReturn(1L); + when(host.getName()).thenReturn("test-host"); + + Map> accessDetails = new HashMap<>(); + doReturn(new Pair<>(true, "{\"status\":\"success\",\"power_state\":{\"vm1\":\"PowerOn\",\"vm2\":\"PowerOff\"}}")) + .when(provisioner).getInstanceStatusesOnExternalSystem(anyString(), anyString(), anyString(), anyMap(), anyInt()); + + Map result = provisioner.getVmPowerStates(host, accessDetails, "test-extension", "test-path"); + + assertNotNull(result); + assertEquals(2, result.size()); + assertEquals(VirtualMachine.PowerState.PowerOn, result.get("vm1").getState()); + assertEquals(VirtualMachine.PowerState.PowerOff, result.get("vm2").getState()); + } + + @Test + public void getVmPowerStatesReturnsNullWhenResponseIsFailure() { + Host host = mock(Host.class); + when(host.getName()).thenReturn("test-host"); + + Map> accessDetails = new HashMap<>(); + doReturn(new Pair<>(false, "Error")).when(provisioner) + .getInstanceStatusesOnExternalSystem(anyString(), anyString(), anyString(), anyMap(), anyInt()); + + Map result = provisioner.getVmPowerStates(host, accessDetails, "test-extension", "test-path"); + assertNull(result); + } + @Test public void getVirtualMachineTOReturnsValidTOWhenVmIsNotNull() { VirtualMachine vm = mock(VirtualMachine.class); @@ -986,4 +1018,120 @@ public class ExternalPathPayloadProvisionerTest { String result = provisioner.getExtensionConfigureError("test-extension", null); assertEquals("Extension: test-extension not configured", result); } + + @Test + public void getVmPowerStatesReturnsNullWhenResponseIsEmpty() { + Host host = mock(Host.class); + when(host.getName()).thenReturn("test-host"); + + Map> accessDetails = new HashMap<>(); + doReturn(new Pair<>(true, "")).when(provisioner) + .getInstanceStatusesOnExternalSystem(anyString(), anyString(), anyString(), anyMap(), anyInt()); + + Map result = provisioner.getVmPowerStates(host, accessDetails, "test-extension", "test-path"); + + assertNull(result); + } + + @Test + public void getVmPowerStatesReturnsNullWhenResponseHasInvalidStatus() { + Host host = mock(Host.class); + when(host.getName()).thenReturn("test-host"); + + Map> accessDetails = new HashMap<>(); + doReturn(new Pair<>(true, "{\"status\":\"failure\"}")).when(provisioner) + .getInstanceStatusesOnExternalSystem(anyString(), anyString(), anyString(), anyMap(), anyInt()); + + Map result = provisioner.getVmPowerStates(host, accessDetails, "test-extension", "test-path"); + + assertNull(result); + } + + @Test + public void getVmPowerStatesReturnsNullWhenPowerStateIsMissing() { + Host host = mock(Host.class); + when(host.getName()).thenReturn("test-host"); + + Map> accessDetails = new HashMap<>(); + doReturn(new Pair<>(true, "{\"status\":\"success\"}")).when(provisioner) + .getInstanceStatusesOnExternalSystem(anyString(), anyString(), anyString(), anyMap(), anyInt()); + + Map result = provisioner.getVmPowerStates(host, accessDetails, "test-extension", "test-path"); + + assertNull(result); + } + + @Test + public void getVmPowerStatesReturnsNullWhenResponseIsMalformed() { + Host host = mock(Host.class); + when(host.getName()).thenReturn("test-host"); + + Map> accessDetails = new HashMap<>(); + doReturn(new Pair<>(true, "{status:success")).when(provisioner) + .getInstanceStatusesOnExternalSystem(anyString(), anyString(), anyString(), anyMap(), anyInt()); + + Map result = provisioner.getVmPowerStates(host, accessDetails, "test-extension", "test-path"); + + assertNull(result); + } + + @Test + public void getInstanceStatusesOnExternalSystemReturnsSuccessWhenCommandExecutesSuccessfully() { + doReturn(new Pair<>(true, "success")).when(provisioner) + .executeExternalCommand(eq("test-extension"), eq("statuses"), anyMap(), eq(30), anyString(), eq("test-file")); + + Pair result = provisioner.getInstanceStatusesOnExternalSystem( + "test-extension", "test-file", "test-host", new HashMap<>(), 30); + + assertTrue(result.first()); + assertEquals("success", result.second()); + } + + @Test + public void getInstanceStatusesOnExternalSystemReturnsFailureWhenCommandFails() { + doReturn(new Pair<>(false, "error")).when(provisioner) + .executeExternalCommand(eq("test-extension"), eq("statuses"), anyMap(), eq(30), anyString(), eq("test-file")); + + Pair result = provisioner.getInstanceStatusesOnExternalSystem( + "test-extension", "test-file", "test-host", new HashMap<>(), 30); + + assertFalse(result.first()); + assertEquals("error", result.second()); + } + + @Test + public void getInstanceStatusesOnExternalSystemHandlesEmptyResponse() { + doReturn(new Pair<>(true, "")).when(provisioner) + .executeExternalCommand(eq("test-extension"), eq("statuses"), anyMap(), eq(30), anyString(), eq("test-file")); + + Pair result = provisioner.getInstanceStatusesOnExternalSystem( + "test-extension", "test-file", "test-host", new HashMap<>(), 30); + + assertTrue(result.first()); + assertEquals("", result.second()); + } + + @Test + public void getInstanceStatusesOnExternalSystemHandlesNullResponse() { + doReturn(new Pair<>(true, null)).when(provisioner) + .executeExternalCommand(eq("test-extension"), eq("statuses"), anyMap(), eq(30), anyString(), eq("test-file")); + + Pair result = provisioner.getInstanceStatusesOnExternalSystem( + "test-extension", "test-file", "test-host", new HashMap<>(), 30); + + assertTrue(result.first()); + assertNull(result.second()); + } + + @Test + public void getInstanceStatusesOnExternalSystemHandlesInvalidFilePath() { + doReturn(new Pair<>(false, "File not found")).when(provisioner) + .executeExternalCommand(eq("test-extension"), eq("statuses"), anyMap(), eq(30), anyString(), eq("invalid-file")); + + Pair result = provisioner.getInstanceStatusesOnExternalSystem( + "test-extension", "invalid-file", "test-host", new HashMap<>(), 30); + + assertFalse(result.first()); + assertEquals("File not found", result.second()); + } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java index 1635ab03837..da60f6fd717 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java @@ -236,7 +236,7 @@ public class BridgeVifDriver extends VifDriverBase { String brName = createVnetBr(vNetId, trafficLabel, protocol); intf.defBridgeNet(brName, null, nic.getMac(), getGuestNicModel(guestOsType, nicAdapter), networkRateKBps); } else { - String brName = createVnetBr(vNetId, _bridges.get("private"), protocol); + String brName = createVnetBr(vNetId, _bridges.get("guest"), protocol); intf.defBridgeNet(brName, null, nic.getMac(), getGuestNicModel(guestOsType, nicAdapter), networkRateKBps); } } else { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 5941f15e63d..0a9e0d2d98e 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -1971,7 +1971,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv for (final String ifNamePattern : ifNamePatterns) { commonPattern.append("|(").append(ifNamePattern).append(".*)"); } - if(fname.matches(commonPattern.toString())) { + if (fname.matches(commonPattern.toString())) { return true; } return false; @@ -2498,11 +2498,10 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv final Pattern pattern = Pattern.compile("(\\D+)(\\d+)(\\D*)(\\d*)(\\D*)(\\d*)"); final Matcher matcher = pattern.matcher(pif); LOGGER.debug("getting broadcast uri for pif " + pif + " and bridge " + brName); - if(matcher.find()) { + if (matcher.find()) { if (brName.startsWith("brvx")){ return BroadcastDomainType.Vxlan.toUri(matcher.group(2)).toString(); - } - else{ + } else { if (!matcher.group(6).isEmpty()) { return BroadcastDomainType.Vlan.toUri(matcher.group(6)).toString(); } else if (!matcher.group(4).isEmpty()) { @@ -3742,7 +3741,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv } else if (volume.getType() == Volume.Type.DATADISK) { final KVMPhysicalDisk physicalDisk = storagePoolManager.getPhysicalDisk(store.getPoolType(), store.getUuid(), data.getPath()); final KVMStoragePool pool = physicalDisk.getPool(); - if(StoragePoolType.RBD.equals(pool.getType())) { + if (StoragePoolType.RBD.equals(pool.getType())) { final int devId = volume.getDiskSeq().intValue(); final String device = mapRbdDevice(physicalDisk); if (device != null) { @@ -4787,12 +4786,11 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv String dataDiskController = details.get(VmDetailConstants.DATA_DISK_CONTROLLER); if (StringUtils.isNotBlank(dataDiskController)) { - LOGGER.debug("Passed custom disk controller for DATA disk " + dataDiskController); - for (DiskDef.DiskBus bus : DiskDef.DiskBus.values()) { - if (bus.toString().equalsIgnoreCase(dataDiskController)) { - LOGGER.debug("Found matching enum for disk controller for DATA disk " + dataDiskController); - return bus; - } + LOGGER.debug("Passed custom disk controller for DATA disk {}", dataDiskController); + DiskDef.DiskBus bus = DiskDef.DiskBus.fromValue(dataDiskController); + if (bus != null) { + LOGGER.debug("Found matching enum for disk controller for DATA disk {}", dataDiskController); + return bus; } } return null; @@ -5201,7 +5199,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv } for (int i = 0; i < memoryStats.length; i++) { - if(memoryStats[i].getTag() == UNUSEDMEMORY) { + if (memoryStats[i].getTag() == UNUSEDMEMORY) { freeMemory = memoryStats[i].getValue(); break; } @@ -5773,12 +5771,12 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv return hypervisorType; } - public String mapRbdDevice(final KVMPhysicalDisk disk){ + public String mapRbdDevice(final KVMPhysicalDisk disk) { final KVMStoragePool pool = disk.getPool(); //Check if rbd image is already mapped final String[] splitPoolImage = disk.getPath().split("/"); String device = Script.runSimpleBashScript("rbd showmapped | grep \""+splitPoolImage[0]+"[ ]*"+splitPoolImage[1]+"\" | grep -o \"[^ ]*[ ]*$\""); - if(device == null) { + if (device == null) { //If not mapped, map and return mapped device Script.runSimpleBashScript("rbd map " + disk.getPath() + " --id " + pool.getAuthUserName()); device = Script.runSimpleBashScript("rbd showmapped | grep \""+splitPoolImage[0]+"[ ]*"+splitPoolImage[1]+"\" | grep -o \"[^ ]*[ ]*$\""); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtDomainXMLParser.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtDomainXMLParser.java index 8b26bc94e21..4d823783a99 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtDomainXMLParser.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtDomainXMLParser.java @@ -111,7 +111,9 @@ public class LibvirtDomainXMLParser { def.defNetworkBasedDisk(diskPath, host, port, authUserName, poolUuid, diskLabel, DiskDef.DiskBus.valueOf(bus.toUpperCase()), DiskDef.DiskProtocol.valueOf(protocol.toUpperCase()), fmt); - def.setCacheMode(DiskDef.DiskCacheMode.valueOf(diskCacheMode.toUpperCase())); + if (StringUtils.isNotBlank(diskCacheMode)) { + def.setCacheMode(DiskDef.DiskCacheMode.valueOf(diskCacheMode.toUpperCase())); + } } else { String diskFmtType = getAttrValue("driver", "type", disk); String diskCacheMode = getAttrValue("driver", "cache", disk); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java index 0234d4b240e..097a9b8dd32 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java @@ -860,6 +860,15 @@ public class LibvirtVMDef { _bus = bus; } + public static DiskBus fromValue(String bus) { + for (DiskBus b : DiskBus.values()) { + if (b.toString().equalsIgnoreCase(bus)) { + return b; + } + } + return null; + } + @Override public String toString() { return _bus; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java index 81328d6ffb9..43607edc53a 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java @@ -163,7 +163,7 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper dpdkPortsMapping = command.getDpdkInterfaceMapping(); if (MapUtils.isNotEmpty(dpdkPortsMapping)) { if (logger.isTraceEnabled()) { - logger.trace(String.format("Changing VM [%s] DPDK interfaces during migration to host: [%s].", vmName, target)); + logger.trace("Changing VM {} DPDK interfaces during migration to host: {}.", vmName, target); } xmlDesc = replaceDpdkInterfaces(xmlDesc, dpdkPortsMapping); if (logger.isDebugEnabled()) { - logger.debug(String.format("Changed VM [%s] XML configuration of DPDK interfaces. New XML configuration is [%s].", vmName, xmlDesc)); + logger.debug("Changed VM {} XML configuration of DPDK interfaces. New XML configuration is {}.", vmName, maskSensitiveInfoInXML(xmlDesc)); } } @@ -240,7 +240,7 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper migrateThread = executor.submit(worker); executor.shutdown(); long sleeptime = 0; + final int migrateDowntime = libvirtComputingResource.getMigrateDowntime(); + boolean isMigrateDowntimeSet = false; + while (!executor.isTerminated()) { Thread.sleep(100); sleeptime += 100; - if (sleeptime == 1000) { // wait 1s before attempting to set downtime on migration, since I don't know of a VIR_DOMAIN_MIGRATING state - final int migrateDowntime = libvirtComputingResource.getMigrateDowntime(); - if (migrateDowntime > 0 ) { - try { - final int setDowntime = dm.migrateSetMaxDowntime(migrateDowntime); - if (setDowntime == 0 ) { - logger.debug("Set max downtime for migration of " + vmName + " to " + String.valueOf(migrateDowntime) + "ms"); - } - } catch (final LibvirtException e) { - logger.debug("Failed to set max downtime for migration, perhaps migration completed? Error: " + e.getMessage()); + if (!isMigrateDowntimeSet && migrateDowntime > 0 && sleeptime >= 1000) { // wait 1s before attempting to set downtime on migration, since I don't know of a VIR_DOMAIN_MIGRATING state + try { + final int setDowntime = dm.migrateSetMaxDowntime(migrateDowntime); + if (setDowntime == 0 ) { + isMigrateDowntimeSet = true; + logger.debug("Set max downtime for migration of " + vmName + " to " + String.valueOf(migrateDowntime) + "ms"); } + } catch (final LibvirtException e) { + logger.debug("Failed to set max downtime for migration, perhaps migration completed? Error: " + e.getMessage()); } } if (sleeptime % 1000 == 0) { @@ -287,7 +288,7 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper]*type=['\"]vnc['\"][^>]*passwd=['\"])([^'\"]*)(['\"])", + "$1*****$3"); + } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartCommandWrapper.java index 09e4ec38027..56798646590 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartCommandWrapper.java @@ -84,8 +84,9 @@ public final class LibvirtStartCommandWrapper extends CommandWrapper details) + final Long iopsWriteRateMaxLength, final String cacheMode, final DiskDef.LibvirtDiskEncryptDetails encryptDetails, Map details, Map controllerInfo) throws LibvirtException, InternalErrorException { attachOrDetachDisk(conn, attach, vmName, attachingDisk, devId, serial, bytesReadRate, bytesReadRateMax, bytesReadRateMaxLength, bytesWriteRate, bytesWriteRateMax, bytesWriteRateMaxLength, iopsReadRate, iopsReadRateMax, iopsReadRateMaxLength, iopsWriteRate, - iopsWriteRateMax, iopsWriteRateMaxLength, cacheMode, encryptDetails, 0l, details); + iopsWriteRateMax, iopsWriteRateMaxLength, cacheMode, encryptDetails, 0l, details, controllerInfo); } /** * * Attaches or detaches a disk to an instance. - * @param conn libvirt connection - * @param attach boolean that determines whether the device will be attached or detached - * @param vmName instance name - * @param attachingDisk kvm physical disk - * @param devId device id in instance + * @param conn libvirt connection + * @param attach boolean that determines whether the device will be attached or detached + * @param vmName instance name + * @param attachingDisk kvm physical disk + * @param devId device id in instance * @param serial - * @param bytesReadRate bytes read rate - * @param bytesReadRateMax bytes read rate max - * @param bytesReadRateMaxLength bytes read rate max length - * @param bytesWriteRate bytes write rate - * @param bytesWriteRateMax bytes write rate amx + * @param bytesReadRate bytes read rate + * @param bytesReadRateMax bytes read rate max + * @param bytesReadRateMaxLength bytes read rate max length + * @param bytesWriteRate bytes write rate + * @param bytesWriteRateMax bytes write rate amx * @param bytesWriteRateMaxLength bytes write rate max length - * @param iopsReadRate iops read rate - * @param iopsReadRateMax iops read rate max - * @param iopsReadRateMaxLength iops read rate max length - * @param iopsWriteRate iops write rate - * @param iopsWriteRateMax iops write rate max - * @param iopsWriteRateMaxLength iops write rate max length - * @param cacheMode cache mode - * @param encryptDetails encrypt details - * @param waitDetachDevice value set in milliseconds to wait before assuming device removal failed + * @param iopsReadRate iops read rate + * @param iopsReadRateMax iops read rate max + * @param iopsReadRateMaxLength iops read rate max length + * @param iopsWriteRate iops write rate + * @param iopsWriteRateMax iops write rate max + * @param iopsWriteRateMaxLength iops write rate max length + * @param cacheMode cache mode + * @param encryptDetails encrypt details + * @param waitDetachDevice value set in milliseconds to wait before assuming device removal failed + * @param controllerInfo * @throws LibvirtException * @throws InternalErrorException */ @@ -1440,7 +1451,7 @@ public class KVMStorageProcessor implements StorageProcessor { final Long bytesWriteRate, final Long bytesWriteRateMax, final Long bytesWriteRateMaxLength, final Long iopsReadRate, final Long iopsReadRateMax, final Long iopsReadRateMaxLength, final Long iopsWriteRate, final Long iopsWriteRateMax, final Long iopsWriteRateMaxLength, final String cacheMode, final DiskDef.LibvirtDiskEncryptDetails encryptDetails, - long waitDetachDevice, Map details) + long waitDetachDevice, Map details, Map controllerInfo) throws LibvirtException, InternalErrorException { List disks = null; @@ -1458,7 +1469,7 @@ public class KVMStorageProcessor implements StorageProcessor { if (resource.getHypervisorType() == Hypervisor.HypervisorType.LXC) { final String device = resource.mapRbdDevice(attachingDisk); if (device != null) { - logger.debug("RBD device on host is: "+device); + logger.debug("RBD device on host is: " + device); attachingDisk.setPath(device); } } @@ -1477,17 +1488,7 @@ public class KVMStorageProcessor implements StorageProcessor { return; } } else { - DiskDef.DiskBus busT = DiskDef.DiskBus.VIRTIO; - for (final DiskDef disk : disks) { - if (disk.getDeviceType() == DeviceType.DISK) { - if (disk.getBusType() == DiskDef.DiskBus.SCSI) { - busT = DiskDef.DiskBus.SCSI; - } else if (disk.getBusType() == DiskDef.DiskBus.VIRTIOBLK) { - busT = DiskDef.DiskBus.VIRTIOBLK; - } - break; - } - } + DiskDef.DiskBus busT = getAttachDiskBusType(devId, disks, controllerInfo); diskdef = new DiskDef(); if (busT == DiskDef.DiskBus.SCSI || busT == DiskDef.DiskBus.VIRTIOBLK) { diskdef.setQemuDriver(true); @@ -1495,11 +1496,11 @@ public class KVMStorageProcessor implements StorageProcessor { } diskdef.setSerial(serial); if (attachingPool.getType() == StoragePoolType.RBD) { - if(resource.getHypervisorType() == Hypervisor.HypervisorType.LXC){ + if (resource.getHypervisorType() == Hypervisor.HypervisorType.LXC) { // For LXC, map image to host and then attach to Vm final String device = resource.mapRbdDevice(attachingDisk); if (device != null) { - logger.debug("RBD device on host is: "+device); + logger.debug("RBD device on host is: " + device); diskdef.defBlockBasedDisk(device, devId, busT); } else { throw new InternalErrorException("Error while mapping disk "+attachingDisk.getPath()+" on host"); @@ -1569,7 +1570,7 @@ public class KVMStorageProcessor implements StorageProcessor { if ((iopsWriteRateMaxLength != null) && (iopsWriteRateMaxLength > 0)) { diskdef.setIopsWriteRateMaxLength(iopsWriteRateMaxLength); } - if(cacheMode != null) { + if (cacheMode != null) { diskdef.setCacheMode(DiskDef.DiskCacheMode.valueOf(cacheMode.toUpperCase())); } @@ -1592,6 +1593,28 @@ public class KVMStorageProcessor implements StorageProcessor { } } + protected DiskDef.DiskBus getAttachDiskBusType(int deviceId, List disks, Map controllerInfo) { + String controllerKey = deviceId == 0 ? VmDetailConstants.ROOT_DISK_CONTROLLER : VmDetailConstants.DATA_DISK_CONTROLLER; + String diskController = MapUtils.getString(controllerInfo, controllerKey); + DiskDef.DiskBus busType = DiskDef.DiskBus.fromValue(diskController); + if (diskController != null) { + logger.debug("Using controller '{}' from command specified as {} while attaching disk (deviceId={})", + diskController, controllerKey, deviceId); + return busType; + } + for (final DiskDef disk : disks) { + if (disk.getDeviceType() != DeviceType.DISK) { + continue; + } + if (disk.getBusType() == DiskDef.DiskBus.SCSI) { + return DiskDef.DiskBus.SCSI; + } else if (disk.getBusType() == DiskDef.DiskBus.VIRTIOBLK) { + return DiskDef.DiskBus.VIRTIOBLK; + } + } + return DiskDef.DiskBus.VIRTIO; + } + @Override public Answer attachVolume(final AttachCommand cmd) { final DiskTO disk = cmd.getDisk(); @@ -1619,7 +1642,8 @@ public class KVMStorageProcessor implements StorageProcessor { vol.getBytesReadRate(), vol.getBytesReadRateMax(), vol.getBytesReadRateMaxLength(), vol.getBytesWriteRate(), vol.getBytesWriteRateMax(), vol.getBytesWriteRateMaxLength(), vol.getIopsReadRate(), vol.getIopsReadRateMax(), vol.getIopsReadRateMaxLength(), - vol.getIopsWriteRate(), vol.getIopsWriteRateMax(), vol.getIopsWriteRateMaxLength(), volCacheMode, encryptDetails, disk.getDetails()); + vol.getIopsWriteRate(), vol.getIopsWriteRateMax(), vol.getIopsWriteRateMaxLength(), volCacheMode, + encryptDetails, disk.getDetails(), cmd.getControllerInfo()); resource.recreateCheckpointsOnVm(List.of((VolumeObjectTO) disk.getData()), vmName, conn); @@ -1658,7 +1682,7 @@ public class KVMStorageProcessor implements StorageProcessor { vol.getBytesReadRate(), vol.getBytesReadRateMax(), vol.getBytesReadRateMaxLength(), vol.getBytesWriteRate(), vol.getBytesWriteRateMax(), vol.getBytesWriteRateMaxLength(), vol.getIopsReadRate(), vol.getIopsReadRateMax(), vol.getIopsReadRateMaxLength(), - vol.getIopsWriteRate(), vol.getIopsWriteRateMax(), vol.getIopsWriteRateMaxLength(), volCacheMode, null, waitDetachDevice, null); + vol.getIopsWriteRate(), vol.getIopsWriteRateMax(), vol.getIopsWriteRateMaxLength(), volCacheMode, null, waitDetachDevice, null, null); storagePoolMgr.disconnectPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), vol.getPath()); @@ -1733,7 +1757,7 @@ public class KVMStorageProcessor implements StorageProcessor { } final VolumeObjectTO newVol = new VolumeObjectTO(); - if(vol != null) { + if (vol != null) { newVol.setPath(vol.getName()); if (vol.getQemuEncryptFormat() != null) { newVol.setEncryptFormat(vol.getQemuEncryptFormat().toString()); @@ -1838,8 +1862,11 @@ public class KVMStorageProcessor implements StorageProcessor { } } else { if (primaryPool.getType() == StoragePoolType.RBD) { - takeRbdVolumeSnapshotOfStoppedVm(primaryPool, disk, snapshotName); + Long snapshotSize = takeRbdVolumeSnapshotOfStoppedVm(primaryPool, disk, snapshotName); newSnapshot.setPath(snapshotPath); + if (snapshotSize != null) { + newSnapshot.setPhysicalSize(snapshotSize); + } } else if (primaryPool.getType() == StoragePoolType.CLVM) { CreateObjectAnswer result = takeClvmVolumeSnapshotOfStoppedVm(disk, snapshotName); if (result != null) return result; @@ -2291,7 +2318,8 @@ public class KVMStorageProcessor implements StorageProcessor { * barriers properly (>2.6.32) this won't be any different then pulling the power * cord out of a running machine. */ - private void takeRbdVolumeSnapshotOfStoppedVm(KVMStoragePool primaryPool, KVMPhysicalDisk disk, String snapshotName) { + private Long takeRbdVolumeSnapshotOfStoppedVm(KVMStoragePool primaryPool, KVMPhysicalDisk disk, String snapshotName) { + Long snapshotSize = null; try { Rados r = radosConnect(primaryPool); @@ -2302,11 +2330,43 @@ public class KVMStorageProcessor implements StorageProcessor { logger.debug("Attempting to create RBD snapshot {}@{}", disk.getName(), snapshotName); image.snapCreate(snapshotName); + image.snapCreate(snapshotName); + long rbdSnapshotSize = getRbdSnapshotSize(primaryPool.getSourceDir(), disk.getName(), snapshotName, primaryPool.getSourceHost(), primaryPool.getAuthUserName(), primaryPool.getAuthSecret()); + if (rbdSnapshotSize > 0) { + snapshotSize = rbdSnapshotSize; + } + rbd.close(image); r.ioCtxDestroy(io); } catch (final Exception e) { logger.error("A RBD snapshot operation on [{}] failed. The error was: {}", disk.getName(), e.getMessage(), e); } + return snapshotSize; + } + + private long getRbdSnapshotSize(String poolPath, String diskName, String snapshotName, String rbdMonitor, String authUser, String authSecret) { + logger.debug("Get RBD snapshot size for {}/{}@{}", poolPath, diskName, snapshotName); + //cmd: rbd du /@ --format json --mon-host --id --key 2>/dev/null + String snapshotDetailsInJson = Script.runSimpleBashScript(String.format("rbd du %s/%s@%s --format json --mon-host %s --id %s --key %s 2>/dev/null", poolPath, diskName, snapshotName, rbdMonitor, authUser, authSecret)); + if (StringUtils.isNotBlank(snapshotDetailsInJson)) { + ObjectMapper mapper = new ObjectMapper(); + try { + JsonNode root = mapper.readTree(snapshotDetailsInJson); + for (JsonNode image : root.path("images")) { + if (snapshotName.equals(image.path("snapshot").asText())) { + long usedSizeInBytes = image.path("used_size").asLong(); + logger.debug("RBD snapshot {}/{}@{} used size in bytes: {}", poolPath, diskName, snapshotName, usedSizeInBytes); + return usedSizeInBytes; + } + } + } catch (JsonProcessingException e) { + logger.error("Unable to get the RBD snapshot size, RBD snapshot cmd output: {}", snapshotDetailsInJson, e); + } + } else { + logger.warn("Failed to get RBD snapshot size for {}/{}@{} - no output for RBD snapshot cmd", poolPath, diskName, snapshotName); + } + + return 0; } /** diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapperTest.java index 0407ed8bbfd..aa36fb89765 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapperTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapperTest.java @@ -658,7 +658,7 @@ public class LibvirtMigrateCommandWrapperTest { @Test public void testReplaceIpForVNCInDescFile() { final String targetIp = "192.168.22.21"; - final String result = libvirtMigrateCmdWrapper.replaceIpForVNCInDescFileAndNormalizePassword(fullfile, targetIp, null, ""); + final String result = libvirtMigrateCmdWrapper.replaceIpForVNCInDescFileAndNormalizePassword(fullfile, targetIp, "vncSecretPwd", ""); assertEquals("transformation does not live up to expectation:\n" + result, targetfile, result); } @@ -1089,6 +1089,30 @@ public class LibvirtMigrateCommandWrapperTest { Assert.assertTrue(finalXml.contains(newIsoVolumePath)); } + @Test + public void testMaskVncPwdDomain() { + // Test case 1: Single quotes + String xml1 = ""; + String expected1 = ""; + assertEquals(expected1, LibvirtMigrateCommandWrapper.maskSensitiveInfoInXML(xml1)); + + // Test case 2: Double quotes + String xml2 = ""; + String expected2 = ""; + assertEquals(expected2, LibvirtMigrateCommandWrapper.maskSensitiveInfoInXML(xml2)); + + // Test case 3: Non-VNC graphics (should remain unchanged) + String xml3 = ""; + assertEquals(xml3, LibvirtMigrateCommandWrapper.maskSensitiveInfoInXML(xml3)); + + // Test case 4: Multiple VNC entries in one string + String xml4 = "\n" + + ""; + String expected4 = "\n" + + ""; + assertEquals(expected4, LibvirtMigrateCommandWrapper.maskSensitiveInfoInXML(xml4)); + } + @Test public void updateGpuDevicesIfNeededTestNoGpuDevice() throws Exception { Mockito.doReturn(virtualMachineTOMock).when(migrateCommandMock).getVirtualMachine(); diff --git a/plugins/hypervisors/ovm/src/main/java/com/cloud/ovm/hypervisor/OvmResourceBase.java b/plugins/hypervisors/ovm/src/main/java/com/cloud/ovm/hypervisor/OvmResourceBase.java index 951741cbb59..51455429039 100644 --- a/plugins/hypervisors/ovm/src/main/java/com/cloud/ovm/hypervisor/OvmResourceBase.java +++ b/plugins/hypervisors/ovm/src/main/java/com/cloud/ovm/hypervisor/OvmResourceBase.java @@ -362,7 +362,7 @@ public class OvmResourceBase implements ServerResource, HypervisorResource { sshConnection = SSHCmdHelper.acquireAuthorizedConnection(_ip, _username, _password); if (sshConnection == null) { - throw new CloudRuntimeException(String.format("Cannot connect to ovm host(IP=%1$s, username=%2$s, password=%3$s", _ip, _username, _password)); + throw new CloudRuntimeException(String.format("Cannot connect to ovm host(IP=%1$s, username=%2$s)", _ip, _username)); } if (!SSHCmdHelper.sshExecuteCmd(sshConnection, "sh /usr/bin/configureOvm.sh postSetup")) { diff --git a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java index 063a5a18ca2..cdb4d7434ae 100644 --- a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java +++ b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java @@ -51,6 +51,7 @@ import java.util.concurrent.TimeoutException; import javax.naming.ConfigurationException; import javax.xml.parsers.ParserConfigurationException; +import com.xensource.xenapi.VTPM; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.diagnostics.CopyToSecondaryStorageAnswer; import org.apache.cloudstack.diagnostics.CopyToSecondaryStorageCommand; @@ -5826,4 +5827,82 @@ public abstract class CitrixResourceBase extends ServerResourceBase implements S public void destroyVm(VM vm, Connection connection) throws XenAPIException, XmlRpcException { destroyVm(vm, connection, false); } + + /** + * Configure vTPM (Virtual Trusted Platform Module) support for a VM. + * vTPM provides a virtual TPM 2.0 device for VMs, enabling features like Secure Boot and disk encryption. + * + * Requirements: + * - XenServer/XCP-ng 8.3 (and above) + * - UEFI Secure Boot enabled + * - VM in halted state + * + * @param conn XenServer connection + * @param vm The VM to configure + * @param vmSpec VM specification containing vTPM settings + */ + public void configureVTPM(Connection conn, VM vm, VirtualMachineTO vmSpec) throws XenAPIException, XmlRpcException { + if (vmSpec == null || vmSpec.getDetails() == null) { + return; + } + + String vtpmEnabled = vmSpec.getDetails().getOrDefault(VmDetailConstants.VIRTUAL_TPM_ENABLED, null); + + final Map platform = vm.getPlatform(conn); + if (platform != null) { + final String guestRequiresVtpm = platform.get("vtpm"); + if (guestRequiresVtpm != null && Boolean.parseBoolean(guestRequiresVtpm) && !Boolean.parseBoolean(vtpmEnabled)) { + logger.warn("Guest OS requires vTPM by default, even if VM details doesn't have the setting: {}", vmSpec.getName()); + return; + } + } + + if (!Boolean.parseBoolean(vtpmEnabled)) { + return; + } + + String bootMode = StringUtils.defaultIfEmpty(vmSpec.getDetails().get(ApiConstants.BootType.UEFI.toString()), null); + String bootType = (bootMode == null) ? ApiConstants.BootType.BIOS.toString() : ApiConstants.BootType.UEFI.toString(); + + if (!ApiConstants.BootType.UEFI.toString().equals(bootType)) { + logger.warn("vTPM requires UEFI boot mode. Skipping vTPM configuration for VM: {}", vmSpec.getName()); + return; + } + + try { + Set existingVtpms = vm.getVTPMs(conn); + if (!existingVtpms.isEmpty()) { + logger.debug("vTPM already exists for VM: {}", vmSpec.getName()); + return; + } + + // Creates vTPM using: xe vtpm-create vm-uuid= + String vmUuid = vm.getUuid(conn); + String result = callHostPlugin(conn, "vmops", "create_vtpm", "vm_uuid", vmUuid); + + if (result == null || result.isEmpty() || result.startsWith("ERROR:") || result.startsWith("EXCEPTION:")) { + throw new CloudRuntimeException("Failed to create vTPM, result: " + result); + } + + logger.info("Successfully created vTPM {} for VM: {}", result.trim(), vmSpec.getName()); + } catch (Exception e) { + logger.warn("Failed to configure vTPM for VM: {}, continuing without vTPM", vmSpec.getName(), e); + } + } + + public boolean isVTPMSupported(Connection conn, Host host) { + try { + Host.Record hostRecord = host.getRecord(conn); + String productVersion = hostRecord.softwareVersion.get("product_version"); + if (productVersion == null) { + return false; + } + ComparableVersion currentVersion = new ComparableVersion(productVersion); + ComparableVersion minVersion = new ComparableVersion("8.2.0"); + return currentVersion.compareTo(minVersion) >= 0; + } catch (Exception e) { + logger.warn("Failed to check vTPM support on host", e); + return false; + } + } } diff --git a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixReadyCommandWrapper.java b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixReadyCommandWrapper.java index c5605e85f94..4f55ae82337 100644 --- a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixReadyCommandWrapper.java +++ b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixReadyCommandWrapper.java @@ -60,12 +60,20 @@ public final class CitrixReadyCommandWrapper extends CommandWrapper vms = host.getResidentVMs(conn); citrixResourceBase.destroyPatchVbd(conn, vms); + } catch (final Exception e) { + logger.warn("Unable to destroy CD-ROM device for system VMs", e); + } + + try { + final Host host = Host.getByUuid(conn, citrixResourceBase.getHost().getUuid()); final Host.Record hr = host.getRecord(conn); if (isUefiSupported(CitrixHelper.getProductVersion(hr))) { hostDetails.put(com.cloud.host.Host.HOST_UEFI_ENABLE, Boolean.TRUE.toString()); } - } catch (final Exception e) { + } catch (Exception e) { + logger.warn("Unable to get UEFI support info", e); } + try { final boolean result = citrixResourceBase.cleanupHaltedVms(conn); if (!result) { diff --git a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixStartCommandWrapper.java b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixStartCommandWrapper.java index d448638f028..5c2355a4cec 100644 --- a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixStartCommandWrapper.java +++ b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixStartCommandWrapper.java @@ -97,6 +97,14 @@ public final class CitrixStartCommandWrapper extends CommandWrapper result; try { - logger.debug("Executing command in Host: " + cmdLine); + logger.debug("Executing password update command on host: {} for user: {}", hostIp, username); final String hostPassword = citrixResourceBase.getPwdFromQueue(); - result = xenServerUtilitiesHelper.executeSshWrapper(hostIp, 22, username, null, hostPassword, cmdLine.toString()); + result = xenServerUtilitiesHelper.executeSshWrapper(hostIp, 22, username, null, hostPassword, cmdLine); } catch (final Exception e) { return new Answer(command, false, e.getMessage()); } diff --git a/plugins/integrations/cloudian/src/main/java/org/apache/cloudstack/cloudian/api/CloudianIsEnabledCmd.java b/plugins/integrations/cloudian/src/main/java/org/apache/cloudstack/cloudian/api/CloudianIsEnabledCmd.java index 56cb74e3cab..3c334ba55c2 100644 --- a/plugins/integrations/cloudian/src/main/java/org/apache/cloudstack/cloudian/api/CloudianIsEnabledCmd.java +++ b/plugins/integrations/cloudian/src/main/java/org/apache/cloudstack/cloudian/api/CloudianIsEnabledCmd.java @@ -31,7 +31,8 @@ import com.cloud.user.Account; responseObject = CloudianEnabledResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, since = "4.11.0", - authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}, + httpMethod = "GET") public class CloudianIsEnabledCmd extends BaseCmd { @Inject diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index e6ed850fba5..d19470f8bab 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -477,7 +477,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne logger.warn("Network offering: {} does not have necessary services to provision Kubernetes cluster", networkOffering); return false; } - if (!networkOffering.isEgressDefaultPolicy()) { + if (!networkOffering.isForVpc() && !networkOffering.isEgressDefaultPolicy()) { logger.warn("Network offering: {} has egress default policy turned off should be on to provision Kubernetes cluster", networkOffering); return false; } @@ -1172,9 +1172,12 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne CallContext networkContext = CallContext.register(CallContext.current(), ApiCommandResourceType.Network); try { + Long zoneId = zone.getId(); + Integer publicMTU = NetworkService.VRPublicInterfaceMtu.valueIn(zoneId); + Integer privateMTU = NetworkService.VRPrivateInterfaceMtu.valueIn(zoneId); network = networkService.createGuestNetwork(networkOffering.getId(), clusterName + "-network", - owner.getAccountName() + "-network", owner, physicalNetwork, zone.getId(), - ControlledEntity.ACLType.Account); + owner.getAccountName() + "-network", owner, physicalNetwork, zoneId, + ControlledEntity.ACLType.Account, new Pair<>(publicMTU, privateMTU)); if (!networkOffering.isForVpc() && NetworkOffering.RoutingMode.Dynamic == networkOffering.getRoutingMode()) { bgpService.allocateASNumber(zone.getId(), asNumber, network.getId(), null); } @@ -1383,8 +1386,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne } totalAdditionalVms += additional; - long effectiveCpu = (long) so.getCpu() * so.getSpeed(); - totalAdditionalCpuUnits += effectiveCpu * additional; + totalAdditionalCpuUnits += so.getCpu() * additional; totalAdditionalRamMb += so.getRamSize() * additional; try { diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java index 99f826402ce..aef020335f2 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java @@ -37,6 +37,9 @@ import org.apache.cloudstack.api.response.GetUploadParamsResponse; import org.apache.cloudstack.api.response.KubernetesSupportedVersionResponse; import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; +import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import com.cloud.api.query.dao.TemplateJoinDao; @@ -57,6 +60,7 @@ import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VMTemplateZoneDao; import com.cloud.template.TemplateApiService; import com.cloud.template.VirtualMachineTemplate; +import com.cloud.user.Account; import com.cloud.user.AccountManager; import com.cloud.utils.Pair; import com.cloud.utils.component.ComponentContext; @@ -84,12 +88,14 @@ public class KubernetesVersionManagerImpl extends ManagerBase implements Kuberne @Inject private DataCenterDao dataCenterDao; @Inject + private ImageStoreDao imageStoreDao; + @Inject private TemplateApiService templateService; public static final String MINIMUN_AUTOSCALER_SUPPORTED_VERSION = "1.15.0"; protected void updateTemplateDetailsInKubernetesSupportedVersionResponse( - final KubernetesSupportedVersion kubernetesSupportedVersion, KubernetesSupportedVersionResponse response) { + final KubernetesSupportedVersion kubernetesSupportedVersion, KubernetesSupportedVersionResponse response, boolean isRootAdmin) { TemplateJoinVO template = templateJoinDao.findById(kubernetesSupportedVersion.getIsoId()); if (template == null) { return; @@ -99,11 +105,14 @@ public class KubernetesVersionManagerImpl extends ManagerBase implements Kuberne if (template.getState() != null) { response.setIsoState(template.getState().toString()); } + if (isRootAdmin) { + response.setIsoUrl(template.getUrl()); + } response.setIsoArch(template.getArch().getType()); response.setDirectDownload(template.isDirectDownload()); } - private KubernetesSupportedVersionResponse createKubernetesSupportedVersionResponse(final KubernetesSupportedVersion kubernetesSupportedVersion) { + private KubernetesSupportedVersionResponse createKubernetesSupportedVersionResponse(final KubernetesSupportedVersion kubernetesSupportedVersion, boolean isRootAdmin) { KubernetesSupportedVersionResponse response = new KubernetesSupportedVersionResponse(); response.setObjectName("kubernetessupportedversion"); response.setId(kubernetesSupportedVersion.getUuid()); @@ -122,7 +131,7 @@ public class KubernetesVersionManagerImpl extends ManagerBase implements Kuberne response.setSupportsHA(compareSemanticVersions(kubernetesSupportedVersion.getSemanticVersion(), KubernetesClusterService.MIN_KUBERNETES_VERSION_HA_SUPPORT)>=0); response.setSupportsAutoscaling(versionSupportsAutoscaling(kubernetesSupportedVersion)); - updateTemplateDetailsInKubernetesSupportedVersionResponse(kubernetesSupportedVersion, response); + updateTemplateDetailsInKubernetesSupportedVersionResponse(kubernetesSupportedVersion, response, isRootAdmin); response.setCreated(kubernetesSupportedVersion.getCreated()); return response; } @@ -130,8 +139,11 @@ public class KubernetesVersionManagerImpl extends ManagerBase implements Kuberne private ListResponse createKubernetesSupportedVersionListResponse( List versions, Integer count) { List responseList = new ArrayList<>(); + Account caller = CallContext.current().getCallingAccount(); + boolean isRootAdmin = accountManager.isRootAdmin(caller.getId()); + for (KubernetesSupportedVersionVO version : versions) { - responseList.add(createKubernetesSupportedVersionResponse(version)); + responseList.add(createKubernetesSupportedVersionResponse(version, isRootAdmin)); } ListResponse response = new ListResponse<>(); response.setResponses(responseList, count); @@ -347,6 +359,32 @@ public class KubernetesVersionManagerImpl extends ManagerBase implements Kuberne return createKubernetesSupportedVersionListResponse(versions, versionsAndCount.second()); } + private void validateImageStoreForZone(Long zoneId, boolean directDownload) { + if (directDownload) { + return; + } + if (zoneId != null) { + List imageStores = imageStoreDao.listStoresByZoneId(zoneId); + if (CollectionUtils.isEmpty(imageStores)) { + DataCenterVO zone = dataCenterDao.findById(zoneId); + String zoneName = zone != null ? zone.getName() : String.valueOf(zoneId); + throw new InvalidParameterValueException(String.format("Unable to register Kubernetes version ISO. No image store available in zone: %s", zoneName)); + } + } else { + List zones = dataCenterDao.listAllZones(); + List zonesWithoutStorage = new ArrayList<>(); + for (DataCenterVO zone : zones) { + List imageStores = imageStoreDao.listStoresByZoneId(zone.getId()); + if (CollectionUtils.isEmpty(imageStores)) { + zonesWithoutStorage.add(zone.getName()); + } + } + if (!zonesWithoutStorage.isEmpty()) { + throw new InvalidParameterValueException(String.format("Unable to register Kubernetes version ISO for all zones. The following zones have no image store: %s", String.join(", ", zonesWithoutStorage))); + } + } + } + private void validateKubernetesSupportedVersion(Long zoneId, String semanticVersion, Integer minimumCpu, Integer minimumRamSize, boolean isDirectDownload) { if (minimumCpu == null || minimumCpu < KubernetesClusterService.MIN_KUBERNETES_CLUSTER_NODE_CPU) { @@ -398,6 +436,8 @@ public class KubernetesVersionManagerImpl extends ManagerBase implements Kuberne } } + validateImageStoreForZone(zoneId, isDirectDownload); + VMTemplateVO template = null; try { VirtualMachineTemplate vmTemplate = registerKubernetesVersionIso(zoneId, name, isoUrl, isoChecksum, isDirectDownload, arch); @@ -411,7 +451,7 @@ public class KubernetesVersionManagerImpl extends ManagerBase implements Kuberne supportedVersionVO = kubernetesSupportedVersionDao.persist(supportedVersionVO); CallContext.current().putContextParameter(KubernetesSupportedVersion.class, supportedVersionVO.getUuid()); - return createKubernetesSupportedVersionResponse(supportedVersionVO); + return createKubernetesSupportedVersionResponse(supportedVersionVO, true); } @Override @@ -496,7 +536,7 @@ public class KubernetesVersionManagerImpl extends ManagerBase implements Kuberne } version = kubernetesSupportedVersionDao.findById(versionId); } - return createKubernetesSupportedVersionResponse(version); + return createKubernetesSupportedVersionResponse(version, true); } @Override diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java index 9ce6dc3ea78..d30922269da 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java @@ -388,7 +388,7 @@ public class CreateKubernetesClusterCmd extends BaseAsyncCreateCmd { @Override public long getEntityOwnerId() { - Long accountId = _accountService.finalyzeAccountId(accountName, domainId, projectId, true); + Long accountId = _accountService.finalizeAccountId(accountName, domainId, projectId, true); if (accountId == null) { return CallContext.current().getCallingAccount().getId(); } diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesSupportedVersionResponse.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesSupportedVersionResponse.java index cfa3212e409..f6e1ee85944 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesSupportedVersionResponse.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesSupportedVersionResponse.java @@ -50,6 +50,10 @@ public class KubernetesSupportedVersionResponse extends BaseResponse { @Param(description = "The name of the binaries ISO for Kubernetes supported version") private String isoName; + @SerializedName(ApiConstants.ISO_URL) + @Param(description = "the URL of the binaries ISO for Kubernetes supported version") + private String isoUrl; + @SerializedName(ApiConstants.ISO_STATE) @Param(description = "The state of the binaries ISO for Kubernetes supported version") private String isoState; @@ -134,6 +138,14 @@ public class KubernetesSupportedVersionResponse extends BaseResponse { this.isoName = isoName; } + public String getIsoUrl() { + return isoUrl; + } + + public void setIsoUrl(String isoUrl) { + this.isoUrl = isoUrl; + } + public String getIsoState() { return isoState; } diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/version/KubernetesVersionManagerImplTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/version/KubernetesVersionManagerImplTest.java index f827610c3cb..35f8e66e045 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/version/KubernetesVersionManagerImplTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/version/KubernetesVersionManagerImplTest.java @@ -16,10 +16,15 @@ // under the License. package com.cloud.kubernetes.version; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; import java.util.UUID; import org.apache.cloudstack.api.response.KubernetesSupportedVersionResponse; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; +import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; +import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -32,6 +37,9 @@ import org.springframework.test.util.ReflectionTestUtils; import com.cloud.api.query.dao.TemplateJoinDao; import com.cloud.api.query.vo.TemplateJoinVO; import com.cloud.cpu.CPU; +import com.cloud.dc.DataCenterVO; +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.exception.InvalidParameterValueException; @RunWith(MockitoJUnitRunner.class) public class KubernetesVersionManagerImplTest { @@ -39,6 +47,12 @@ public class KubernetesVersionManagerImplTest { @Mock TemplateJoinDao templateJoinDao; + @Mock + ImageStoreDao imageStoreDao; + + @Mock + DataCenterDao dataCenterDao; + @InjectMocks KubernetesVersionManagerImpl kubernetesVersionManager = new KubernetesVersionManagerImpl(); @@ -48,7 +62,7 @@ public class KubernetesVersionManagerImplTest { Mockito.when(kubernetesSupportedVersion.getIsoId()).thenReturn(1L); KubernetesSupportedVersionResponse response = new KubernetesSupportedVersionResponse(); kubernetesVersionManager.updateTemplateDetailsInKubernetesSupportedVersionResponse(kubernetesSupportedVersion, - response); + response, true); Assert.assertNull(ReflectionTestUtils.getField(response, "isoId")); } @@ -63,13 +77,71 @@ public class KubernetesVersionManagerImplTest { Mockito.when(templateJoinVO.getUuid()).thenReturn(uuid); Mockito.when(templateJoinDao.findById(1L)).thenReturn(templateJoinVO); kubernetesVersionManager.updateTemplateDetailsInKubernetesSupportedVersionResponse(kubernetesSupportedVersion, - response); + response, true); Assert.assertEquals(uuid, ReflectionTestUtils.getField(response, "isoId")); Assert.assertNull(ReflectionTestUtils.getField(response, "isoState")); ObjectInDataStoreStateMachine.State state = ObjectInDataStoreStateMachine.State.Ready; Mockito.when(templateJoinVO.getState()).thenReturn(state); kubernetesVersionManager.updateTemplateDetailsInKubernetesSupportedVersionResponse(kubernetesSupportedVersion, - response); + response, true); Assert.assertEquals(state.toString(), ReflectionTestUtils.getField(response, "isoState")); } + + @Test + public void testValidateImageStoreForZoneWithDirectDownload() { + ReflectionTestUtils.invokeMethod(kubernetesVersionManager, "validateImageStoreForZone", 1L, true); + } + + @Test + public void testValidateImageStoreForZoneWithValidZone() { + Long zoneId = 1L; + List imageStores = Collections.singletonList(Mockito.mock(ImageStoreVO.class)); + Mockito.when(imageStoreDao.listStoresByZoneId(zoneId)).thenReturn(imageStores); + + ReflectionTestUtils.invokeMethod(kubernetesVersionManager, "validateImageStoreForZone", zoneId, false); + } + + @Test(expected = InvalidParameterValueException.class) + public void testValidateImageStoreForZoneWithNoImageStore() { + Long zoneId = 1L; + DataCenterVO zone = Mockito.mock(DataCenterVO.class); + Mockito.when(zone.getName()).thenReturn("test-zone"); + Mockito.when(dataCenterDao.findById(zoneId)).thenReturn(zone); + Mockito.when(imageStoreDao.listStoresByZoneId(zoneId)).thenReturn(Collections.emptyList()); + + ReflectionTestUtils.invokeMethod(kubernetesVersionManager, "validateImageStoreForZone", zoneId, false); + } + + @Test + public void testValidateImageStoreForAllZonesWithAllValid() { + DataCenterVO zone1 = Mockito.mock(DataCenterVO.class); + Mockito.when(zone1.getId()).thenReturn(1L); + DataCenterVO zone2 = Mockito.mock(DataCenterVO.class); + Mockito.when(zone2.getId()).thenReturn(2L); + List zones = Arrays.asList(zone1, zone2); + Mockito.when(dataCenterDao.listAllZones()).thenReturn(zones); + + List imageStores = Collections.singletonList(Mockito.mock(ImageStoreVO.class)); + Mockito.when(imageStoreDao.listStoresByZoneId(1L)).thenReturn(imageStores); + Mockito.when(imageStoreDao.listStoresByZoneId(2L)).thenReturn(imageStores); + + ReflectionTestUtils.invokeMethod(kubernetesVersionManager, "validateImageStoreForZone", (Long) null, false); + } + + @Test(expected = InvalidParameterValueException.class) + public void testValidateImageStoreForAllZonesWithSomeMissingStorage() { + DataCenterVO zone1 = Mockito.mock(DataCenterVO.class); + Mockito.when(zone1.getId()).thenReturn(1L); + DataCenterVO zone2 = Mockito.mock(DataCenterVO.class); + Mockito.when(zone2.getId()).thenReturn(2L); + Mockito.when(zone2.getName()).thenReturn("zone-without-storage"); + List zones = Arrays.asList(zone1, zone2); + Mockito.when(dataCenterDao.listAllZones()).thenReturn(zones); + + List imageStores = Collections.singletonList(Mockito.mock(ImageStoreVO.class)); + Mockito.when(imageStoreDao.listStoresByZoneId(1L)).thenReturn(imageStores); + Mockito.when(imageStoreDao.listStoresByZoneId(2L)).thenReturn(Collections.emptyList()); + + ReflectionTestUtils.invokeMethod(kubernetesVersionManager, "validateImageStoreForZone", (Long) null, false); + } } diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/version/KubernetesVersionServiceTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/version/KubernetesVersionServiceTest.java index 455df6b57d4..7ba35169b9e 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/version/KubernetesVersionServiceTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/version/KubernetesVersionServiceTest.java @@ -17,6 +17,9 @@ package com.cloud.kubernetes.version; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertNotNull; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.when; import java.lang.reflect.Field; @@ -25,6 +28,11 @@ import java.util.List; import java.util.UUID; import com.cloud.cpu.CPU; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; +import com.cloud.user.AccountVO; +import com.cloud.user.User; +import com.cloud.user.UserVO; import org.apache.cloudstack.api.command.admin.kubernetes.version.AddKubernetesSupportedVersionCmd; import org.apache.cloudstack.api.command.admin.kubernetes.version.DeleteKubernetesSupportedVersionCmd; import org.apache.cloudstack.api.command.admin.kubernetes.version.UpdateKubernetesSupportedVersionCmd; @@ -63,11 +71,6 @@ import com.cloud.storage.VMTemplateVO; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.template.TemplateApiService; import com.cloud.template.VirtualMachineTemplate; -import com.cloud.user.Account; -import com.cloud.user.AccountManager; -import com.cloud.user.AccountVO; -import com.cloud.user.User; -import com.cloud.user.UserVO; import com.cloud.utils.Pair; import com.cloud.utils.component.ComponentContext; import com.cloud.utils.db.Filter; @@ -75,6 +78,9 @@ import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; +import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; + @RunWith(MockitoJUnitRunner.class) public class KubernetesVersionServiceTest { @@ -94,7 +100,11 @@ public class KubernetesVersionServiceTest { @Mock private DataCenterDao dataCenterDao; @Mock + private ImageStoreDao imageStoreDao; + @Mock private TemplateApiService templateService; + @Mock + private Account accountMock; AutoCloseable closeable; @@ -123,7 +133,12 @@ public class KubernetesVersionServiceTest { DataCenterVO zone = Mockito.mock(DataCenterVO.class); when(dataCenterDao.findById(Mockito.anyLong())).thenReturn(zone); + List imageStores = new ArrayList<>(); + imageStores.add(Mockito.mock(ImageStoreVO.class)); + when(imageStoreDao.listStoresByZoneId(Mockito.anyLong())).thenReturn(imageStores); + TemplateJoinVO templateJoinVO = Mockito.mock(TemplateJoinVO.class); + when(templateJoinVO.getUrl()).thenReturn("https://download.cloudstack.com"); when(templateJoinVO.getState()).thenReturn(ObjectInDataStoreStateMachine.State.Ready); when(templateJoinVO.getArch()).thenReturn(CPU.CPUArch.getDefault()); when(templateJoinDao.findById(Mockito.anyLong())).thenReturn(templateJoinVO); @@ -140,19 +155,66 @@ public class KubernetesVersionServiceTest { @Test public void listKubernetesSupportedVersionsTest() { - ListKubernetesSupportedVersionsCmd cmd = Mockito.mock(ListKubernetesSupportedVersionsCmd.class); - List versionVOs = new ArrayList<>(); - KubernetesSupportedVersionVO versionVO = Mockito.mock(KubernetesSupportedVersionVO.class); - when(versionVO.getSemanticVersion()).thenReturn(KubernetesVersionService.MIN_KUBERNETES_VERSION); - versionVOs.add(versionVO); - when(kubernetesSupportedVersionDao.findById(Mockito.anyLong())).thenReturn(versionVO); - when(kubernetesSupportedVersionDao.searchAndCount(Mockito.any(SearchCriteria.class), - Mockito.any(Filter.class))).thenReturn(new Pair<>(versionVOs, versionVOs.size())); - ListResponse versionsResponse = - kubernetesVersionService.listKubernetesSupportedVersions(cmd); - Assert.assertEquals(versionVOs.size(), versionsResponse.getCount().intValue()); - Assert.assertTrue(CollectionUtils.isNotEmpty(versionsResponse.getResponses())); - Assert.assertEquals(versionVOs.size(), versionsResponse.getResponses().size()); + CallContext callContextMock = Mockito.mock(CallContext.class); + try (MockedStatic callContextMockedStatic = Mockito.mockStatic(CallContext.class)) { + callContextMockedStatic.when(CallContext::current).thenReturn(callContextMock); + final SearchCriteria versionSearchCriteria = Mockito.mock(SearchCriteria.class); + when(callContextMock.getCallingAccount()).thenReturn(accountMock); + ListKubernetesSupportedVersionsCmd cmd = Mockito.mock(ListKubernetesSupportedVersionsCmd.class); + List versionVOs = new ArrayList<>(); + KubernetesSupportedVersionVO versionVO = Mockito.mock(KubernetesSupportedVersionVO.class); + when(versionVO.getSemanticVersion()).thenReturn(KubernetesVersionService.MIN_KUBERNETES_VERSION); + versionVOs.add(versionVO); + when(kubernetesSupportedVersionDao.findById(Mockito.anyLong())).thenReturn(versionVO); + when(kubernetesSupportedVersionDao.searchAndCount(Mockito.any(), Mockito.any(Filter.class))) + .thenReturn(new Pair<>(versionVOs, versionVOs.size())); + ListResponse versionsResponse = + kubernetesVersionService.listKubernetesSupportedVersions(cmd); + Assert.assertEquals(versionVOs.size(), versionsResponse.getCount().intValue()); + Assert.assertTrue(CollectionUtils.isNotEmpty(versionsResponse.getResponses())); + Assert.assertEquals(versionVOs.size(), versionsResponse.getResponses().size()); + } + } + + @Test + public void listKubernetesSupportedVersionsTestWhenAdmin() { + CallContext callContextMock = Mockito.mock(CallContext.class); + try (MockedStatic callContextMockedStatic = Mockito.mockStatic(CallContext.class)) { + callContextMockedStatic.when(CallContext::current).thenReturn(callContextMock); + ListKubernetesSupportedVersionsCmd cmd = Mockito.mock(ListKubernetesSupportedVersionsCmd.class); + List versionVOs = new ArrayList<>(); + KubernetesSupportedVersionVO versionVO = Mockito.mock(KubernetesSupportedVersionVO.class); + when(versionVO.getSemanticVersion()).thenReturn(KubernetesVersionService.MIN_KUBERNETES_VERSION); + versionVOs.add(versionVO); + when(callContextMock.getCallingAccount()).thenReturn(accountMock); + when(kubernetesSupportedVersionDao.findById(Mockito.anyLong())).thenReturn(versionVO); + when(kubernetesSupportedVersionDao.searchAndCount(Mockito.any(), Mockito.any(Filter.class))) + .thenReturn(new Pair<>(versionVOs, versionVOs.size())); + when(accountManager.isRootAdmin(anyLong())).thenReturn(true); + ListResponse response = kubernetesVersionService.listKubernetesSupportedVersions(cmd); + assertNotNull(response.getResponses().get(0).getIsoUrl()); + } + } + + @Test + public void listKubernetesSupportedVersionsTestWhenOtherUser() { + CallContext callContextMock = Mockito.mock(CallContext.class); + try (MockedStatic callContextMockedStatic = Mockito.mockStatic(CallContext.class)) { + callContextMockedStatic.when(CallContext::current).thenReturn(callContextMock); + ListKubernetesSupportedVersionsCmd cmd = Mockito.mock(ListKubernetesSupportedVersionsCmd.class); + List versionVOs = new ArrayList<>(); + KubernetesSupportedVersionVO versionVO = Mockito.mock(KubernetesSupportedVersionVO.class); + when(versionVO.getSemanticVersion()).thenReturn(KubernetesVersionService.MIN_KUBERNETES_VERSION); + versionVOs.add(versionVO); + when(callContextMock.getCallingAccount()).thenReturn(accountMock); + when(kubernetesSupportedVersionDao.findById(Mockito.anyLong())).thenReturn(versionVO); + when(kubernetesSupportedVersionDao.searchAndCount(Mockito.any(), Mockito.any(Filter.class))) + .thenReturn(new Pair<>(versionVOs, versionVOs.size())); + when(accountManager.isRootAdmin(anyLong())).thenReturn(false); + when(accountMock.getId()).thenReturn(2L); + ListResponse response = kubernetesVersionService.listKubernetesSupportedVersions(cmd); + assertNull(response.getResponses().get(0).getIsoUrl()); + } } @Test(expected = InvalidParameterValueException.class) @@ -224,7 +286,6 @@ public class KubernetesVersionServiceTest { mockedComponentContext.when(() -> ComponentContext.inject(Mockito.any(RegisterIsoCmd.class))).thenReturn( new RegisterIsoCmd()); mockedCallContext.when(CallContext::current).thenReturn(callContext); - when(templateService.registerIso(Mockito.any(RegisterIsoCmd.class))).thenReturn( Mockito.mock(VirtualMachineTemplate.class)); VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class); diff --git a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java index 8908bfa000f..2ed65efd1e8 100644 --- a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java +++ b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java @@ -512,20 +512,48 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp public String getMetrics() { final StringBuilder stringBuilder = new StringBuilder(); stringBuilder.append("# Cloudstack Prometheus Metrics\n"); - for (final Item item : metricsItems) { + + List sortedItems = metricsItems.stream() + .sorted((item1, item2) -> item1.name.compareTo(item2.name)) + .collect(Collectors.toList()); + + String currentMetricName = null; + + for (Item item : sortedItems) { + if (!item.name.equals(currentMetricName)) { + currentMetricName = item.name; + stringBuilder.append("# HELP ").append(currentMetricName).append(" ") + .append(item.getHelp()).append("\n"); + stringBuilder.append("# TYPE ").append(currentMetricName).append(" ") + .append(item.getType()).append("\n"); + } + stringBuilder.append(item.toMetricsString()).append("\n"); } + return stringBuilder.toString(); } private abstract class Item { String name; + String help; + String type; - public Item(final String nm) { + public Item(final String nm, final String hlp, final String tp) { name = nm; + help = hlp; + type = tp; } public abstract String toMetricsString(); + + public String getHelp() { + return help; + } + + public String getType() { + return type; + } } class ItemVM extends Item { @@ -535,7 +563,9 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp long total; public ItemVM(final String zn, final String zu, final String st, long cnt) { - super("cloudstack_vms_total"); + super("cloudstack_vms_total", + "Total number of virtual machines", + "gauge"); zoneName = zn; zoneUuid = zu; filter = st; @@ -556,7 +586,9 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp String hosttags; public ItemVMByTag(final String zn, final String zu, final String st, long cnt, final String tags) { - super("cloudstack_vms_total_by_tag"); + super("cloudstack_vms_total_by_tag", + "Total number of virtual machines grouped by host tags", + "gauge"); zoneName = zn; zoneUuid = zu; filter = st; @@ -577,7 +609,9 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp int total; public ItemVolume(final String zn, final String zu, final String st, int cnt) { - super("cloudstack_volumes_total"); + super("cloudstack_volumes_total", + "Total number of volumes", + "gauge"); zoneName = zn; zoneUuid = zu; filter = st; @@ -598,7 +632,9 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp String hosttags; public ItemHost(final String zn, final String zu, final String st, int cnt, final String tags) { - super("cloudstack_hosts_total"); + super("cloudstack_hosts_total", + "Total number of hosts", + "gauge"); zoneName = zn; zoneUuid = zu; state = st; @@ -610,6 +646,7 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp public String toMetricsString() { if (StringUtils.isNotEmpty(hosttags)) { name = "cloudstack_hosts_total_by_tag"; + help = "Total number of hosts grouped by tags"; return String.format("%s{zone=\"%s\",filter=\"%s\",tags=\"%s\"} %d", name, zoneName, state, hosttags, total); } return String.format("%s{zone=\"%s\",filter=\"%s\"} %d", name, zoneName, state, total); @@ -628,7 +665,9 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp String hosttags; public ItemVMCore(final String zn, final String zu, final String hn, final String hu, final String hip, final String fl, final Long cr, final int dedicated, final String tags) { - super("cloudstack_host_vms_cores_total"); + super("cloudstack_host_vms_cores_total", + "Total number of VM cores on hosts", + "gauge"); zoneName = zn; zoneUuid = zu; hostName = hn; @@ -649,6 +688,7 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp return String.format("%s{zone=\"%s\",filter=\"%s\"} %d", name, zoneName, filter, core); } else { name = "cloudstack_host_vms_cores_total_by_tag"; + help = "Total number of VM cores grouped by host tags"; return String.format("%s{zone=\"%s\",filter=\"%s\",tags=\"%s\"} %d", name, zoneName, filter, hosttags, core); } } @@ -657,13 +697,14 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp } class MissingHostInfo extends Item { - String zoneName; String hostName; MissingInfoFilter filter; public MissingHostInfo(String zoneName, String hostname, MissingInfoFilter filter) { - super("cloudstack_host_missing_info"); + super("cloudstack_host_missing_info", + "Hosts with missing capacity or statistics information", + "gauge"); this.zoneName = zoneName; this.hostName = hostname; this.filter = filter; @@ -688,7 +729,9 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp String hosttags; public ItemHostCpu(final String zn, final String zu, final String hn, final String hu, final String hip, final String of, final String fl, final double mh, final int dedicated, final String tags) { - super("cloudstack_host_cpu_usage_mhz_total"); + super("cloudstack_host_cpu_usage_mhz_total", + "Host CPU usage in MHz", + "gauge"); zoneName = zn; zoneUuid = zu; hostName = hn; @@ -708,6 +751,7 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp return String.format("%s{zone=\"%s\",filter=\"%s\"} %.2f", name, zoneName, filter, mhertz); } else { name = "cloudstack_host_cpu_usage_mhz_total_by_tag"; + help = "Host CPU usage in MHz grouped by host tags"; return String.format("%s{zone=\"%s\",filter=\"%s\",tags=\"%s\"} %.2f", name, zoneName, filter, hosttags, mhertz); } } @@ -728,7 +772,9 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp String hosttags; public ItemHostMemory(final String zn, final String zu, final String hn, final String hu, final String hip, final String of, final String fl, final double membytes, final int dedicated, final String tags) { - super("cloudstack_host_memory_usage_mibs_total"); + super("cloudstack_host_memory_usage_mibs_total", + "Host memory usage in MiB", + "gauge"); zoneName = zn; zoneUuid = zu; hostName = hn; @@ -748,6 +794,7 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp return String.format("%s{zone=\"%s\",filter=\"%s\"} %.2f", name, zoneName, filter, miBytes); } else { name = "cloudstack_host_memory_usage_mibs_total_by_tag"; + help = "Host memory usage in MiB grouped by host tags"; return String.format("%s{zone=\"%s\",filter=\"%s\",tags=\"%s\"} %.2f", name, zoneName, filter, hosttags, miBytes); } } @@ -764,7 +811,9 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp int total; public ItemHostVM(final String zoneName, final String zoneUuid, final String hostName, final String hostUuid, final String hostIp, final int total) { - super("cloudstack_host_vms_total"); + super("cloudstack_host_vms_total", + "Total number of VMs per host", + "gauge"); this.zoneName = zoneName; this.zoneUuid = zoneUuid; this.hostName = hostName; @@ -790,7 +839,9 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp double total; public ItemPool(final String zn, final String zu, final String pn, final String pa, final String typ, final String of, final String fl, double cnt) { - super("cloudstack_storage_pool_gibs_total"); + super("cloudstack_storage_pool_gibs_total", + "Storage pool capacity in GiB", + "gauge"); zoneName = zn; zoneUuid = zu; pname = pn; @@ -817,7 +868,9 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp int total; public ItemPrivateIp(final String zn, final String zu, final String fl, int cnt) { - super("cloudstack_private_ips_total"); + super("cloudstack_private_ips_total", + "Total number of private IP addresses", + "gauge"); zoneName = zn; zoneUuid = zu; filter = fl; @@ -837,7 +890,9 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp int total; public ItemPublicIp(final String zn, final String zu, final String fl, int cnt) { - super("cloudstack_public_ips_total"); + super("cloudstack_public_ips_total", + "Total number of public IP addresses", + "gauge"); zoneName = zn; zoneUuid = zu; filter = fl; @@ -857,7 +912,9 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp int total; public ItemSharedNetworkIp(final String zn, final String zu, final String fl, int cnt) { - super("cloudstack_shared_network_ips_total"); + super("cloudstack_shared_network_ips_total", + "Total number of shared network IP addresses", + "gauge"); zoneName = zn; zoneUuid = zu; filter = fl; @@ -877,7 +934,9 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp int total; public ItemVlan(final String zn, final String zu, final String fl, int cnt) { - super("cloudstack_vlans_total"); + super("cloudstack_vlans_total", + "Total number of VLANs", + "gauge"); zoneName = zn; zoneUuid = zu; filter = fl; @@ -894,7 +953,9 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp long cores; public ItemDomainLimitCpu(final long c) { - super("cloudstack_domain_limit_cpu_cores_total"); + super("cloudstack_domain_limit_cpu_cores_total", + "Total CPU core limit across all domains", + "gauge"); cores = c; } @@ -908,7 +969,9 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp long miBytes; public ItemDomainLimitMemory(final long mb) { - super("cloudstack_domain_limit_memory_mibs_total"); + super("cloudstack_domain_limit_memory_mibs_total", + "Total memory limit in MiB across all domains", + "gauge"); miBytes = mb; } @@ -927,7 +990,9 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp int isDedicated; public ItemHostIsDedicated(final String zoneName, final String zoneUuid, final String hostName, final String hostUuid, final String hostIp, final int isDedicated) { - super("cloudstack_host_is_dedicated"); + super("cloudstack_host_is_dedicated", + "Whether a host is dedicated (1) or not (0)", + "gauge"); this.zoneName = zoneName; this.zoneUuid = zoneUuid; this.hostName = hostName; @@ -949,7 +1014,9 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp int total; public ItemActiveDomains(final String zn, final String zu, final int cnt) { - super("cloudstack_active_domains_total"); + super("cloudstack_active_domains_total", + "Total number of active domains", + "gauge"); zoneName = zn; zoneUuid = zu; total = cnt; @@ -970,7 +1037,9 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp public ItemHostDedicatedToAccount(final String zoneName, final String hostName, final String accountName, final String domainName, int isDedicated) { - super("cloudstack_host_dedicated_to_account"); + super("cloudstack_host_dedicated_to_account", + "Host dedication to specific account", + "gauge"); this.zoneName = zoneName; this.hostName = hostName; this.accountName = accountName; @@ -991,7 +1060,9 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp String resourceType; public ItemPerDomainResourceLimit(final long c, final String domainName, final String resourceType) { - super("cloudstack_domain_resource_limit"); + super("cloudstack_domain_resource_limit", + "Resource limits per domain", + "gauge"); this.cores = c; this.domainName = domainName; this.resourceType = resourceType; @@ -1009,7 +1080,9 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp String resourceType; public ItemPerDomainResourceCount(final long mb, final String domainName, final String resourceType) { - super("cloudstack_domain_resource_count"); + super("cloudstack_domain_resource_count", + "Resource usage count per domain", + "gauge"); this.miBytes = mb; this.domainName = domainName; this.resourceType = resourceType; @@ -1027,7 +1100,9 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp int total; public ItemActiveAccounts(final String zn, final String zu, final int cnt) { - super("cloudstack_active_accounts_total"); + super("cloudstack_active_accounts_total", + "Total number of active accounts", + "gauge"); zoneName = zn; zoneUuid = zu; total = cnt; @@ -1047,7 +1122,9 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp int total; public ItemVMsBySize(final String zn, final String zu, final int c, final int m, int cnt) { - super("cloudstack_vms_total_by_size"); + super("cloudstack_vms_total_by_size", + "Total number of VMs grouped by CPU and memory size", + "gauge"); zoneName = zn; zoneUuid = zu; cpu = c; diff --git a/plugins/maintenance/src/main/java/org/apache/cloudstack/api/command/ReadyForShutdownCmd.java b/plugins/maintenance/src/main/java/org/apache/cloudstack/api/command/ReadyForShutdownCmd.java index 782b23a0422..36ec4fff9c9 100644 --- a/plugins/maintenance/src/main/java/org/apache/cloudstack/api/command/ReadyForShutdownCmd.java +++ b/plugins/maintenance/src/main/java/org/apache/cloudstack/api/command/ReadyForShutdownCmd.java @@ -26,7 +26,8 @@ import com.cloud.user.Account; description = "Returns the status of CloudStack, whether a shutdown has been triggered and if ready to shutdown", since = "4.19.0", responseObject = ManagementServerMaintenanceResponse.class, - requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, + httpMethod = "GET") public class ReadyForShutdownCmd extends BaseMSMaintenanceActionCmd { public static final String APINAME = "readyForShutdown"; diff --git a/plugins/maintenance/src/test/java/org/apache/cloudstack/maintenance/ManagementServerMaintenanceManagerImplTest.java b/plugins/maintenance/src/test/java/org/apache/cloudstack/maintenance/ManagementServerMaintenanceManagerImplTest.java index 280d1eaf9eb..a208893f6d1 100644 --- a/plugins/maintenance/src/test/java/org/apache/cloudstack/maintenance/ManagementServerMaintenanceManagerImplTest.java +++ b/plugins/maintenance/src/test/java/org/apache/cloudstack/maintenance/ManagementServerMaintenanceManagerImplTest.java @@ -321,7 +321,6 @@ public class ManagementServerMaintenanceManagerImplTest { spy.prepareForMaintenance("static", false); }); - Mockito.when(msHost.getState()).thenReturn(ManagementServerHost.State.Maintenance); Mockito.doNothing().when(jobManagerMock).enableAsyncJobs(); spy.cancelMaintenance(); Mockito.verify(jobManagerMock).enableAsyncJobs(); @@ -339,7 +338,6 @@ public class ManagementServerMaintenanceManagerImplTest { spy.prepareForMaintenance("static", false); }); - Mockito.when(msHost.getState()).thenReturn(ManagementServerHost.State.PreparingForMaintenance); Mockito.doNothing().when(jobManagerMock).enableAsyncJobs(); spy.cancelMaintenance(); Mockito.verify(jobManagerMock).enableAsyncJobs(); diff --git a/plugins/network-elements/globodns/pom.xml b/plugins/network-elements/globodns/pom.xml index 70bf2287e2a..11e84d777c1 100644 --- a/plugins/network-elements/globodns/pom.xml +++ b/plugins/network-elements/globodns/pom.xml @@ -33,8 +33,8 @@ globodns-client - mysql - mysql-connector-java + com.mysql + mysql-connector-j test diff --git a/plugins/network-elements/juniper-contrail/src/main/java/org/apache/cloudstack/network/contrail/api/command/CreateServiceInstanceCmd.java b/plugins/network-elements/juniper-contrail/src/main/java/org/apache/cloudstack/network/contrail/api/command/CreateServiceInstanceCmd.java index db98448ec17..4c0f29c8c8a 100644 --- a/plugins/network-elements/juniper-contrail/src/main/java/org/apache/cloudstack/network/contrail/api/command/CreateServiceInstanceCmd.java +++ b/plugins/network-elements/juniper-contrail/src/main/java/org/apache/cloudstack/network/contrail/api/command/CreateServiceInstanceCmd.java @@ -186,7 +186,7 @@ public class CreateServiceInstanceCmd extends BaseAsyncCreateCmd { @Override public long getEntityOwnerId() { - Long accountId = _accountService.finalyzeAccountId(accountName, domainId, projectId, true); + Long accountId = _accountService.finalizeAccountId(accountName, domainId, projectId, true); if (accountId == null) { return CallContext.current().getCallingAccount().getId(); } diff --git a/plugins/network-elements/juniper-contrail/src/test/java/org/apache/cloudstack/network/contrail/management/MockAccountManager.java b/plugins/network-elements/juniper-contrail/src/test/java/org/apache/cloudstack/network/contrail/management/MockAccountManager.java index 54e76463bca..4dffb405d1a 100644 --- a/plugins/network-elements/juniper-contrail/src/test/java/org/apache/cloudstack/network/contrail/management/MockAccountManager.java +++ b/plugins/network-elements/juniper-contrail/src/test/java/org/apache/cloudstack/network/contrail/management/MockAccountManager.java @@ -467,7 +467,7 @@ public class MockAccountManager extends ManagerBase implements AccountManager { } @Override - public Long finalyzeAccountId(String accountName, Long domainId, Long projectId, boolean enabledOnly) { + public Long finalizeAccountId(String accountName, Long domainId, Long projectId, boolean enabledOnly) { // TODO Auto-generated method stub return null; } diff --git a/plugins/network-elements/nsx/pom.xml b/plugins/network-elements/nsx/pom.xml index e6431d074ca..9c0f680068c 100644 --- a/plugins/network-elements/nsx/pom.xml +++ b/plugins/network-elements/nsx/pom.xml @@ -59,7 +59,7 @@ com.vmware.vapi vapi-runtime - 2.40.0 + 2.61.2 diff --git a/plugins/network-elements/tungsten/pom.xml b/plugins/network-elements/tungsten/pom.xml index eb83c468f0c..12214afebcb 100644 --- a/plugins/network-elements/tungsten/pom.xml +++ b/plugins/network-elements/tungsten/pom.xml @@ -41,8 +41,8 @@ reload4j - mysql - mysql-connector-java + com.mysql + mysql-connector-j test diff --git a/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/driver/DateraPrimaryDataStoreDriver.java b/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/driver/DateraPrimaryDataStoreDriver.java index dcf84525748..62393610499 100644 --- a/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/driver/DateraPrimaryDataStoreDriver.java +++ b/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/driver/DateraPrimaryDataStoreDriver.java @@ -563,7 +563,7 @@ public class DateraPrimaryDataStoreDriver implements PrimaryDataStoreDriver { private long getUsedBytes(StoragePool storagePool, long volumeIdToIgnore) { long usedSpaceBytes = 0; - List lstVolumes = _volumeDao.findByPoolId(storagePool.getId(), null); + List lstVolumes = _volumeDao.findNonDestroyedVolumesByPoolId(storagePool.getId(), null); if (lstVolumes != null) { for (VolumeVO volume : lstVolumes) { diff --git a/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/provider/DateraHostListener.java b/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/provider/DateraHostListener.java index a0dc23da486..08bc89737f2 100644 --- a/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/provider/DateraHostListener.java +++ b/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/provider/DateraHostListener.java @@ -247,7 +247,7 @@ public class DateraHostListener implements HypervisorHostListener { List storagePaths = new ArrayList<>(); // If you do not pass in null for the second parameter, you only get back applicable ROOT disks. - List volumes = _volumeDao.findByPoolId(storagePoolId, null); + List volumes = _volumeDao.findNonDestroyedVolumesByPoolId(storagePoolId, null); if (volumes != null) { for (VolumeVO volume : volumes) { @@ -317,7 +317,7 @@ public class DateraHostListener implements HypervisorHostListener { StoragePoolVO storagePool = _storagePoolDao.findById(storagePoolId); // If you do not pass in null for the second parameter, you only get back applicable ROOT disks. - List volumes = _volumeDao.findByPoolId(storagePoolId, null); + List volumes = _volumeDao.findNonDestroyedVolumesByPoolId(storagePoolId, null); if (volumes != null) { for (VolumeVO volume : volumes) { diff --git a/plugins/storage/volume/linstor/CHANGELOG.md b/plugins/storage/volume/linstor/CHANGELOG.md index 72f0a56697b..1a3142e8c59 100644 --- a/plugins/storage/volume/linstor/CHANGELOG.md +++ b/plugins/storage/volume/linstor/CHANGELOG.md @@ -24,6 +24,12 @@ All notable changes to Linstor CloudStack plugin will be documented in this file The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [2026-01-17] + +### Added + +- Support live migrate from other primary storage + ## [2025-12-18] ### Changed diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java index 67fe4dcc059..3f06bee8ac8 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java @@ -21,33 +21,25 @@ import com.linbit.linstor.api.CloneWaiter; import com.linbit.linstor.api.DevelopersApi; import com.linbit.linstor.api.model.ApiCallRc; import com.linbit.linstor.api.model.ApiCallRcList; -import com.linbit.linstor.api.model.AutoSelectFilter; import com.linbit.linstor.api.model.LayerType; -import com.linbit.linstor.api.model.Properties; import com.linbit.linstor.api.model.ResourceDefinition; import com.linbit.linstor.api.model.ResourceDefinitionCloneRequest; import com.linbit.linstor.api.model.ResourceDefinitionCloneStarted; import com.linbit.linstor.api.model.ResourceDefinitionCreate; import com.linbit.linstor.api.model.ResourceDefinitionModify; -import com.linbit.linstor.api.model.ResourceGroup; -import com.linbit.linstor.api.model.ResourceGroupSpawn; import com.linbit.linstor.api.model.ResourceMakeAvailable; import com.linbit.linstor.api.model.ResourceWithVolumes; import com.linbit.linstor.api.model.Snapshot; import com.linbit.linstor.api.model.SnapshotRestore; -import com.linbit.linstor.api.model.VolumeDefinition; import com.linbit.linstor.api.model.VolumeDefinitionModify; import javax.annotation.Nonnull; -import javax.annotation.Nullable; import javax.inject.Inject; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.stream.Collectors; @@ -117,10 +109,9 @@ import org.apache.cloudstack.storage.snapshot.SnapshotObject; import org.apache.cloudstack.storage.to.SnapshotObjectTO; import org.apache.cloudstack.storage.to.VolumeObjectTO; import org.apache.cloudstack.storage.volume.VolumeObject; - -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.LogManager; import org.apache.commons.collections.CollectionUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.nio.charset.StandardCharsets; @@ -335,275 +326,11 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver } } - private void logLinstorAnswer(@Nonnull ApiCallRc answer) { - if (answer.isError()) { - logger.error(answer.getMessage()); - } else if (answer.isWarning()) { - logger.warn(answer.getMessage()); - } else if (answer.isInfo()) { - logger.info(answer.getMessage()); - } - } - - private void logLinstorAnswers(@Nonnull ApiCallRcList answers) { - answers.forEach(this::logLinstorAnswer); - } - - private void checkLinstorAnswersThrow(@Nonnull ApiCallRcList answers) { - logLinstorAnswers(answers); - if (answers.hasError()) - { - String errMsg = answers.stream() - .filter(ApiCallRc::isError) - .findFirst() - .map(ApiCallRc::getMessage).orElse("Unknown linstor error"); - throw new CloudRuntimeException(errMsg); - } - } - private String checkLinstorAnswers(@Nonnull ApiCallRcList answers) { - logLinstorAnswers(answers); + LinstorUtil.logLinstorAnswers(answers); return answers.stream().filter(ApiCallRc::isError).findFirst().map(ApiCallRc::getMessage).orElse(null); } - private void applyQoSSettings(StoragePoolVO storagePool, DevelopersApi api, String rscName, Long maxIops) - throws ApiException - { - Long currentQosIops = null; - List vlmDfns = api.volumeDefinitionList(rscName, null, null); - if (!vlmDfns.isEmpty()) - { - Properties props = vlmDfns.get(0).getProps(); - long iops = Long.parseLong(props.getOrDefault("sys/fs/blkio_throttle_write_iops", "0")); - currentQosIops = iops > 0 ? iops : null; - } - - if (!Objects.equals(maxIops, currentQosIops)) - { - VolumeDefinitionModify vdm = new VolumeDefinitionModify(); - if (maxIops != null) - { - Properties props = new Properties(); - props.put("sys/fs/blkio_throttle_read_iops", "" + maxIops); - props.put("sys/fs/blkio_throttle_write_iops", "" + maxIops); - vdm.overrideProps(props); - logger.info("Apply qos setting: " + maxIops + " to " + rscName); - } - else - { - logger.info("Remove QoS setting for " + rscName); - vdm.deleteProps(Arrays.asList("sys/fs/blkio_throttle_read_iops", "sys/fs/blkio_throttle_write_iops")); - } - ApiCallRcList answers = api.volumeDefinitionModify(rscName, 0, vdm); - checkLinstorAnswersThrow(answers); - - Long capacityIops = storagePool.getCapacityIops(); - if (capacityIops != null) - { - long vcIops = currentQosIops != null ? currentQosIops * -1 : 0; - long vMaxIops = maxIops != null ? maxIops : 0; - long newIops = vcIops + vMaxIops; - capacityIops -= newIops; - logger.info(String.format("Current storagepool %s iops capacity: %d", storagePool, capacityIops)); - storagePool.setCapacityIops(Math.max(0, capacityIops)); - _storagePoolDao.update(storagePool.getId(), storagePool); - } - } - } - - private String getRscGrp(StoragePool storagePool) { - return storagePool.getUserInfo() != null && !storagePool.getUserInfo().isEmpty() ? - storagePool.getUserInfo() : "DfltRscGrp"; - } - - /** - * Returns the layerlist of the resourceGroup with encryption(LUKS) added above STORAGE. - * If the resourceGroup layer list already contains LUKS this layer list will be returned. - * @param api Linstor developers API - * @param resourceGroup Resource group to get the encryption layer list - * @return layer list with LUKS added - */ - public List getEncryptedLayerList(DevelopersApi api, String resourceGroup) { - try { - List rscGrps = api.resourceGroupList( - Collections.singletonList(resourceGroup), Collections.emptyList(), null, null); - - if (CollectionUtils.isEmpty(rscGrps)) { - throw new CloudRuntimeException( - String.format("Resource Group %s not found on Linstor cluster.", resourceGroup)); - } - - final ResourceGroup rscGrp = rscGrps.get(0); - List layers = Arrays.asList(LayerType.DRBD, LayerType.LUKS, LayerType.STORAGE); - List curLayerStack = rscGrp.getSelectFilter() != null ? - rscGrp.getSelectFilter().getLayerStack() : Collections.emptyList(); - if (CollectionUtils.isNotEmpty(curLayerStack)) { - layers = curLayerStack.stream().map(LayerType::valueOf).collect(Collectors.toList()); - if (!layers.contains(LayerType.LUKS)) { - layers.add(layers.size() - 1, LayerType.LUKS); // lowest layer is STORAGE - } - } - return layers; - } catch (ApiException e) { - throw new CloudRuntimeException( - String.format("Resource Group %s not found on Linstor cluster.", resourceGroup)); - } - } - - /** - * Spawns a new Linstor resource with the given arguments. - * @param api - * @param newRscName - * @param sizeInBytes - * @param isTemplate - * @param rscGrpName - * @param volName - * @param vmName - * @throws ApiException - */ - private void spawnResource( - DevelopersApi api, String newRscName, long sizeInBytes, boolean isTemplate, String rscGrpName, - String volName, String vmName, @Nullable Long passPhraseId, @Nullable byte[] passPhrase) throws ApiException - { - ResourceGroupSpawn rscGrpSpawn = new ResourceGroupSpawn(); - rscGrpSpawn.setResourceDefinitionName(newRscName); - rscGrpSpawn.addVolumeSizesItem(sizeInBytes / 1024); - if (passPhraseId != null) { - AutoSelectFilter asf = new AutoSelectFilter(); - List luksLayers = getEncryptedLayerList(api, rscGrpName); - asf.setLayerStack(luksLayers.stream().map(LayerType::toString).collect(Collectors.toList())); - rscGrpSpawn.setSelectFilter(asf); - if (passPhrase != null) { - String utf8Passphrase = new String(passPhrase, StandardCharsets.UTF_8); - rscGrpSpawn.setVolumePassphrases(Collections.singletonList(utf8Passphrase)); - } - } - - if (isTemplate) { - Properties props = new Properties(); - props.put(LinstorUtil.getTemplateForAuxPropKey(rscGrpName), "true"); - rscGrpSpawn.setResourceDefinitionProps(props); - } - - logger.info("Linstor: Spawn resource " + newRscName); - ApiCallRcList answers = api.resourceGroupSpawn(rscGrpName, rscGrpSpawn); - checkLinstorAnswersThrow(answers); - - answers = LinstorUtil.applyAuxProps(api, newRscName, volName, vmName); - checkLinstorAnswersThrow(answers); - } - - /** - * Condition if a template resource can be shared with the given resource group. - * @param tgtRscGrp - * @param tgtLayerStack - * @param rg - * @return True if the template resource can be shared, else false. - */ - private boolean canShareTemplateForResourceGroup( - ResourceGroup tgtRscGrp, List tgtLayerStack, ResourceGroup rg) { - List rgLayerStack = rg.getSelectFilter() != null ? - rg.getSelectFilter().getLayerStack() : null; - return Objects.equals(tgtLayerStack, rgLayerStack) && - Objects.equals(tgtRscGrp.getSelectFilter().getStoragePoolList(), - rg.getSelectFilter().getStoragePoolList()); - } - - /** - * Searches for a shareable template for this rscGrpName and sets the aux template property. - * @param api - * @param rscName - * @param rscGrpName - * @param existingRDs - * @return - * @throws ApiException - */ - private boolean foundShareableTemplate( - DevelopersApi api, String rscName, String rscGrpName, - List> existingRDs) throws ApiException { - if (!existingRDs.isEmpty()) { - ResourceGroup tgtRscGrp = api.resourceGroupList( - Collections.singletonList(rscGrpName), null, null, null).get(0); - List tgtLayerStack = tgtRscGrp.getSelectFilter() != null ? - tgtRscGrp.getSelectFilter().getLayerStack() : null; - - // check if there is already a template copy, that we could reuse - // this means if select filters are similar enough to allow cloning from - for (Pair rdPair : existingRDs) { - ResourceGroup rg = rdPair.second(); - if (canShareTemplateForResourceGroup(tgtRscGrp, tgtLayerStack, rg)) { - LinstorUtil.setAuxTemplateForProperty(api, rscName, rscGrpName); - return true; - } - } - } - return false; - } - - /** - * Creates a new Linstor resource. - * @param rscName - * @param sizeInBytes - * @param volName - * @param vmName - * @param api - * @param rscGrp - * @param poolId - * @param isTemplate indicates if the resource is a template - * @return true if a new resource was created, false if it already existed or was reused. - */ - private boolean createResourceBase( - String rscName, long sizeInBytes, String volName, String vmName, - @Nullable Long passPhraseId, @Nullable byte[] passPhrase, DevelopersApi api, - String rscGrp, long poolId, boolean isTemplate) - { - try - { - logger.debug("createRscBase: {} :: {} :: {}", rscName, rscGrp, isTemplate); - List> existingRDs = LinstorUtil.getRDAndRGListStartingWith(api, rscName); - - String fullRscName = String.format("%s-%d", rscName, poolId); - boolean alreadyCreated = existingRDs.stream() - .anyMatch(p -> p.first().getName().equalsIgnoreCase(fullRscName)) || - existingRDs.stream().anyMatch(p -> p.first().getProps().containsKey(LinstorUtil.getTemplateForAuxPropKey(rscGrp))); - if (!alreadyCreated) { - boolean createNewRsc = !foundShareableTemplate(api, rscName, rscGrp, existingRDs); - if (createNewRsc) { - String newRscName = existingRDs.isEmpty() ? rscName : fullRscName; - spawnResource(api, newRscName, sizeInBytes, isTemplate, rscGrp, - volName, vmName, passPhraseId, passPhrase); - } - return createNewRsc; - } - return false; - } catch (ApiException apiEx) - { - logger.error("Linstor: ApiEx - " + apiEx.getMessage()); - throw new CloudRuntimeException(apiEx.getBestMessage(), apiEx); - } - } - - private String createResource(VolumeInfo vol, StoragePoolVO storagePoolVO) { - DevelopersApi linstorApi = LinstorUtil.getLinstorAPI(storagePoolVO.getHostAddress()); - final String rscGrp = getRscGrp(storagePoolVO); - - final String rscName = LinstorUtil.RSC_PREFIX + vol.getUuid(); - createResourceBase( - rscName, vol.getSize(), vol.getName(), vol.getAttachedVmName(), vol.getPassphraseId(), vol.getPassphrase(), - linstorApi, rscGrp, storagePoolVO.getId(), false); - - try - { - applyQoSSettings(storagePoolVO, linstorApi, rscName, vol.getMaxIops()); - - return LinstorUtil.getDevicePath(linstorApi, rscName); - } catch (ApiException apiEx) - { - logger.error("Linstor: ApiEx - " + apiEx.getMessage()); - throw new CloudRuntimeException(apiEx.getBestMessage(), apiEx); - } - } - private void resizeResource(DevelopersApi api, String resourceName, long sizeByte) throws ApiException { VolumeDefinitionModify dfm = new VolumeDefinitionModify(); dfm.setSizeKib(sizeByte / 1024); @@ -688,13 +415,14 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver try { ResourceDefinition templateRD = LinstorUtil.findResourceDefinition( - linstorApi, templateRscName, getRscGrp(storagePoolVO)); + linstorApi, templateRscName, LinstorUtil.getRscGrp(storagePoolVO)); final String cloneRes = templateRD != null ? templateRD.getName() : templateRscName; logger.info("Clone resource definition {} to {}", cloneRes, rscName); ResourceDefinitionCloneRequest cloneRequest = new ResourceDefinitionCloneRequest(); cloneRequest.setName(rscName); if (volumeInfo.getPassphraseId() != null) { - List encryptionLayer = getEncryptedLayerList(linstorApi, getRscGrp(storagePoolVO)); + List encryptionLayer = LinstorUtil.getEncryptedLayerList( + linstorApi, LinstorUtil.getRscGrp(storagePoolVO)); cloneRequest.setLayerList(encryptionLayer); if (volumeInfo.getPassphrase() != null) { String utf8Passphrase = new String(volumeInfo.getPassphrase(), StandardCharsets.UTF_8); @@ -704,7 +432,7 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver ResourceDefinitionCloneStarted cloneStarted = linstorApi.resourceDefinitionClone( cloneRes, cloneRequest); - checkLinstorAnswersThrow(cloneStarted.getMessages()); + LinstorUtil.checkLinstorAnswersThrow(cloneStarted.getMessages()); if (!CloneWaiter.waitFor(linstorApi, cloneStarted)) { throw new CloudRuntimeException("Clone for resource " + rscName + " failed."); @@ -716,11 +444,12 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver resizeResource(linstorApi, rscName, volumeInfo.getSize()); } - updateRscGrpIfNecessary(linstorApi, rscName, getRscGrp(storagePoolVO)); + updateRscGrpIfNecessary(linstorApi, rscName, LinstorUtil.getRscGrp(storagePoolVO)); deleteTemplateForProps(linstorApi, rscName); LinstorUtil.applyAuxProps(linstorApi, rscName, volumeInfo.getName(), volumeInfo.getAttachedVmName()); - applyQoSSettings(storagePoolVO, linstorApi, rscName, volumeInfo.getMaxIops()); + LinstorUtil.applyQoSSettings( + _storagePoolDao, storagePoolVO, linstorApi, rscName, volumeInfo.getMaxIops()); return LinstorUtil.getDevicePath(linstorApi, rscName); } catch (ApiException apiEx) { @@ -744,7 +473,7 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver } private String createResourceFromSnapshot(long csSnapshotId, String rscName, StoragePoolVO storagePoolVO) { - final String rscGrp = getRscGrp(storagePoolVO); + final String rscGrp = LinstorUtil.getRscGrp(storagePoolVO); final DevelopersApi linstorApi = LinstorUtil.getLinstorAPI(storagePoolVO.getHostAddress()); SnapshotVO snapshotVO = _snapshotDao.findById(csSnapshotId); @@ -757,22 +486,22 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver logger.debug("Create new resource definition: " + rscName); ResourceDefinitionCreate rdCreate = createResourceDefinitionCreate(rscName, rscGrp); ApiCallRcList answers = linstorApi.resourceDefinitionCreate(rdCreate); - checkLinstorAnswersThrow(answers); + LinstorUtil.checkLinstorAnswersThrow(answers); SnapshotRestore snapshotRestore = new SnapshotRestore(); snapshotRestore.toResource(rscName); logger.debug("Create new volume definition for snapshot: " + cloneRes + ":" + snapName); answers = linstorApi.resourceSnapshotsRestoreVolumeDefinition(cloneRes, snapName, snapshotRestore); - checkLinstorAnswersThrow(answers); + LinstorUtil.checkLinstorAnswersThrow(answers); // restore snapshot to new resource logger.info("Restore resource from snapshot: " + cloneRes + ":" + snapName); answers = linstorApi.resourceSnapshotRestore(cloneRes, snapName, snapshotRestore); - checkLinstorAnswersThrow(answers); + LinstorUtil.checkLinstorAnswersThrow(answers); LinstorUtil.applyAuxProps(linstorApi, rscName, volumeVO.getName(), null); - applyQoSSettings(storagePoolVO, linstorApi, rscName, volumeVO.getMaxIops()); + LinstorUtil.applyQoSSettings(_storagePoolDao, storagePoolVO, linstorApi, rscName, volumeVO.getMaxIops()); return LinstorUtil.getDevicePath(linstorApi, rscName); } catch (ApiException apiEx) { @@ -790,7 +519,7 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver } else if (csTemplateId > 0) { return cloneResource(csTemplateId, volumeInfo, storagePoolVO); } else { - return createResource(volumeInfo, storagePoolVO); + return LinstorUtil.createResource(volumeInfo, storagePoolVO, _storagePoolDao); } } @@ -1141,7 +870,7 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver String rscName, String snapshotName, String restoredName) throws ApiException { - final String rscGrp = getRscGrp(storagePoolVO); + final String rscGrp = LinstorUtil.getRscGrp(storagePoolVO); // try to delete -rst resource, could happen if the copy failed and noone deleted it. deleteResourceDefinition(storagePoolVO, restoredName); ResourceDefinitionCreate rdc = createResourceDefinitionCreate(restoredName, rscGrp); @@ -1186,7 +915,7 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver final StoragePoolVO pool = _storagePoolDao.findById(dstData.getDataStore().getId()); final DevelopersApi api = LinstorUtil.getLinstorAPI(pool.getHostAddress()); final String rscName = LinstorUtil.RSC_PREFIX + dstData.getUuid(); - boolean newCreated = createResourceBase( + boolean newCreated = LinstorUtil.createResourceBase( LinstorUtil.RSC_PREFIX + dstData.getUuid(), tInfo.getSize(), tInfo.getName(), @@ -1194,9 +923,10 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver null, null, api, - getRscGrp(pool), + LinstorUtil.getRscGrp(pool), pool.getId(), - true); + true, + false); Answer answer; if (newCreated) { @@ -1430,7 +1160,7 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver { resizeResource(api, rscName, resizeParameter.newSize); - applyQoSSettings(pool, api, rscName, resizeParameter.newMaxIops); + LinstorUtil.applyQoSSettings(_storagePoolDao, pool, api, rscName, resizeParameter.newMaxIops); { final VolumeVO volume = _volumeDao.findById(vol.getId()); volume.setMinIops(resizeParameter.newMinIops); @@ -1535,7 +1265,7 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver @Override public Pair getStorageStats(StoragePool storagePool) { logger.debug(String.format("Requesting storage stats: %s", storagePool)); - return LinstorUtil.getStorageStats(storagePool.getHostAddress(), getRscGrp(storagePool)); + return LinstorUtil.getStorageStats(storagePool.getHostAddress(), LinstorUtil.getRscGrp(storagePool)); } @Override diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java index 4196c12b116..7c45493dddc 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java @@ -22,6 +22,8 @@ import com.linbit.linstor.api.ApiException; import com.linbit.linstor.api.DevelopersApi; import com.linbit.linstor.api.model.ApiCallRc; import com.linbit.linstor.api.model.ApiCallRcList; +import com.linbit.linstor.api.model.AutoSelectFilter; +import com.linbit.linstor.api.model.LayerType; import com.linbit.linstor.api.model.Node; import com.linbit.linstor.api.model.Properties; import com.linbit.linstor.api.model.ProviderKind; @@ -29,24 +31,36 @@ import com.linbit.linstor.api.model.Resource; import com.linbit.linstor.api.model.ResourceDefinition; import com.linbit.linstor.api.model.ResourceDefinitionModify; import com.linbit.linstor.api.model.ResourceGroup; +import com.linbit.linstor.api.model.ResourceGroupSpawn; import com.linbit.linstor.api.model.ResourceWithVolumes; import com.linbit.linstor.api.model.StoragePool; import com.linbit.linstor.api.model.Volume; +import com.linbit.linstor.api.model.VolumeDefinition; +import com.linbit.linstor.api.model.VolumeDefinitionModify; import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.stream.Collectors; import com.cloud.hypervisor.kvm.storage.KVMStoragePool; import com.cloud.utils.Pair; import com.cloud.utils.exception.CloudRuntimeException; -import org.apache.logging.log4j.Logger; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.commons.collections.CollectionUtils; import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.nio.charset.StandardCharsets; public class LinstorUtil { protected static Logger LOGGER = LogManager.getLogger(LinstorUtil.class); @@ -56,6 +70,8 @@ public class LinstorUtil { public static final String RSC_GROUP = "resourceGroup"; public static final String CS_TEMPLATE_FOR_PREFIX = "_cs-template-for-"; + public static final String LIN_PROP_DRBDOPT_EXACT_SIZE = "DrbdOptions/ExactSize"; + public static final String TEMP_VOLUME_ID = "tempVolumeId"; public static final String CLUSTER_DEFAULT_MIN_IOPS = "clusterDefaultMinIops"; @@ -76,6 +92,32 @@ public class LinstorUtil { .orElse((answers.get(0)).getMessage()) : null; } + public static void logLinstorAnswer(@Nonnull ApiCallRc answer) { + if (answer.isError()) { + LOGGER.error(answer.getMessage()); + } else if (answer.isWarning()) { + LOGGER.warn(answer.getMessage()); + } else if (answer.isInfo()) { + LOGGER.info(answer.getMessage()); + } + } + + public static void logLinstorAnswers(@Nonnull ApiCallRcList answers) { + answers.forEach(LinstorUtil::logLinstorAnswer); + } + + public static void checkLinstorAnswersThrow(@Nonnull ApiCallRcList answers) { + logLinstorAnswers(answers); + if (answers.hasError()) + { + String errMsg = answers.stream() + .filter(ApiCallRc::isError) + .findFirst() + .map(ApiCallRc::getMessage).orElse("Unknown linstor error"); + throw new CloudRuntimeException(errMsg); + } + } + public static List getLinstorNodeNames(@Nonnull DevelopersApi api) throws ApiException { List nodes = api.nodeList( @@ -488,4 +530,253 @@ public class LinstorUtil { } return false; } + + public static String getRscGrp(com.cloud.storage.StoragePool storagePool) { + return storagePool.getUserInfo() != null && !storagePool.getUserInfo().isEmpty() ? + storagePool.getUserInfo() : "DfltRscGrp"; + } + + /** + * Condition if a template resource can be shared with the given resource group. + * @param tgtRscGrp + * @param tgtLayerStack + * @param rg + * @return True if the template resource can be shared, else false. + */ + private static boolean canShareTemplateForResourceGroup( + ResourceGroup tgtRscGrp, List tgtLayerStack, ResourceGroup rg) { + List rgLayerStack = rg.getSelectFilter() != null ? + rg.getSelectFilter().getLayerStack() : null; + return Objects.equals(tgtLayerStack, rgLayerStack) && + Objects.equals(tgtRscGrp.getSelectFilter().getStoragePoolList(), + rg.getSelectFilter().getStoragePoolList()); + } + + /** + * Searches for a shareable template for this rscGrpName and sets the aux template property. + * @param api + * @param rscName + * @param rscGrpName + * @param existingRDs + * @return + * @throws ApiException + */ + private static boolean foundShareableTemplate( + DevelopersApi api, String rscName, String rscGrpName, + List> existingRDs) throws ApiException { + if (!existingRDs.isEmpty()) { + ResourceGroup tgtRscGrp = api.resourceGroupList( + Collections.singletonList(rscGrpName), null, null, null).get(0); + List tgtLayerStack = tgtRscGrp.getSelectFilter() != null ? + tgtRscGrp.getSelectFilter().getLayerStack() : null; + + // check if there is already a template copy, that we could reuse + // this means if select filters are similar enough to allow cloning from + for (Pair rdPair : existingRDs) { + ResourceGroup rg = rdPair.second(); + if (canShareTemplateForResourceGroup(tgtRscGrp, tgtLayerStack, rg)) { + LinstorUtil.setAuxTemplateForProperty(api, rscName, rscGrpName); + return true; + } + } + } + return false; + } + + /** + * Returns the layerlist of the resourceGroup with encryption(LUKS) added above STORAGE. + * If the resourceGroup layer list already contains LUKS this layer list will be returned. + * @param api Linstor developers API + * @param resourceGroup Resource group to get the encryption layer list + * @return layer list with LUKS added + */ + public static List getEncryptedLayerList(DevelopersApi api, String resourceGroup) { + try { + List rscGrps = api.resourceGroupList( + Collections.singletonList(resourceGroup), Collections.emptyList(), null, null); + + if (CollectionUtils.isEmpty(rscGrps)) { + throw new CloudRuntimeException( + String.format("Resource Group %s not found on Linstor cluster.", resourceGroup)); + } + + final ResourceGroup rscGrp = rscGrps.get(0); + List layers = Arrays.asList(LayerType.DRBD, LayerType.LUKS, LayerType.STORAGE); + List curLayerStack = rscGrp.getSelectFilter() != null ? + rscGrp.getSelectFilter().getLayerStack() : Collections.emptyList(); + if (CollectionUtils.isNotEmpty(curLayerStack)) { + layers = curLayerStack.stream().map(LayerType::valueOf).collect(Collectors.toList()); + if (!layers.contains(LayerType.LUKS)) { + layers.add(layers.size() - 1, LayerType.LUKS); // lowest layer is STORAGE + } + } + return layers; + } catch (ApiException e) { + throw new CloudRuntimeException( + String.format("Resource Group %s not found on Linstor cluster.", resourceGroup)); + } + } + + /** + * Spawns a new Linstor resource with the given arguments. + * @param api + * @param newRscName + * @param sizeInBytes + * @param isTemplate + * @param rscGrpName + * @param volName + * @param vmName + * @throws ApiException + */ + private static void spawnResource( + DevelopersApi api, String newRscName, long sizeInBytes, boolean isTemplate, String rscGrpName, + String volName, String vmName, @Nullable Long passPhraseId, @Nullable byte[] passPhrase, + boolean exactSize) throws ApiException + { + ResourceGroupSpawn rscGrpSpawn = new ResourceGroupSpawn(); + rscGrpSpawn.setResourceDefinitionName(newRscName); + rscGrpSpawn.addVolumeSizesItem(sizeInBytes / 1024); + if (passPhraseId != null) { + AutoSelectFilter asf = new AutoSelectFilter(); + List luksLayers = getEncryptedLayerList(api, rscGrpName); + asf.setLayerStack(luksLayers.stream().map(LayerType::toString).collect(Collectors.toList())); + rscGrpSpawn.setSelectFilter(asf); + if (passPhrase != null) { + String utf8Passphrase = new String(passPhrase, StandardCharsets.UTF_8); + rscGrpSpawn.setVolumePassphrases(Collections.singletonList(utf8Passphrase)); + } + } + + Properties props = new Properties(); + if (isTemplate) { + props.put(LinstorUtil.getTemplateForAuxPropKey(rscGrpName), "true"); + } + if (exactSize) { + props.put(LIN_PROP_DRBDOPT_EXACT_SIZE, "true"); + } + rscGrpSpawn.setResourceDefinitionProps(props); + + LOGGER.info("Linstor: Spawn resource " + newRscName); + ApiCallRcList answers = api.resourceGroupSpawn(rscGrpName, rscGrpSpawn); + checkLinstorAnswersThrow(answers); + + answers = LinstorUtil.applyAuxProps(api, newRscName, volName, vmName); + checkLinstorAnswersThrow(answers); + } + + /** + * Creates a new Linstor resource. + * @param rscName + * @param sizeInBytes + * @param volName + * @param vmName + * @param api + * @param rscGrp + * @param poolId + * @param isTemplate indicates if the resource is a template + * @return true if a new resource was created, false if it already existed or was reused. + */ + public static boolean createResourceBase( + String rscName, long sizeInBytes, String volName, String vmName, + @Nullable Long passPhraseId, @Nullable byte[] passPhrase, DevelopersApi api, + String rscGrp, long poolId, boolean isTemplate, boolean exactSize) + { + try + { + LOGGER.debug("createRscBase: {} :: {} :: {} :: {}", rscName, rscGrp, isTemplate, exactSize); + List> existingRDs = LinstorUtil.getRDAndRGListStartingWith(api, rscName); + + String fullRscName = String.format("%s-%d", rscName, poolId); + boolean alreadyCreated = existingRDs.stream() + .anyMatch(p -> p.first().getName().equalsIgnoreCase(fullRscName)) || + existingRDs.stream().anyMatch(p -> p.first().getProps().containsKey(LinstorUtil.getTemplateForAuxPropKey(rscGrp))); + if (!alreadyCreated) { + boolean createNewRsc = !foundShareableTemplate(api, rscName, rscGrp, existingRDs); + if (createNewRsc) { + String newRscName = existingRDs.isEmpty() ? rscName : fullRscName; + spawnResource(api, newRscName, sizeInBytes, isTemplate, rscGrp, + volName, vmName, passPhraseId, passPhrase, exactSize); + } + return createNewRsc; + } + return false; + } catch (ApiException apiEx) + { + LOGGER.error("Linstor: ApiEx - {}", apiEx.getMessage()); + throw new CloudRuntimeException(apiEx.getBestMessage(), apiEx); + } + } + + public static void applyQoSSettings(PrimaryDataStoreDao primaryDataStoreDao, + StoragePoolVO storagePool, DevelopersApi api, String rscName, Long maxIops) + throws ApiException + { + Long currentQosIops = null; + List vlmDfns = api.volumeDefinitionList(rscName, null, null); + if (!vlmDfns.isEmpty()) + { + Properties props = vlmDfns.get(0).getProps(); + long iops = Long.parseLong(props.getOrDefault("sys/fs/blkio_throttle_write_iops", "0")); + currentQosIops = iops > 0 ? iops : null; + } + + if (!Objects.equals(maxIops, currentQosIops)) + { + VolumeDefinitionModify vdm = new VolumeDefinitionModify(); + if (maxIops != null) + { + Properties props = new Properties(); + props.put("sys/fs/blkio_throttle_read_iops", "" + maxIops); + props.put("sys/fs/blkio_throttle_write_iops", "" + maxIops); + vdm.overrideProps(props); + LOGGER.info("Apply qos setting: {} to {}", maxIops, rscName); + } + else + { + LOGGER.info("Remove QoS setting for {}", rscName); + vdm.deleteProps(Arrays.asList("sys/fs/blkio_throttle_read_iops", "sys/fs/blkio_throttle_write_iops")); + } + ApiCallRcList answers = api.volumeDefinitionModify(rscName, 0, vdm); + LinstorUtil.checkLinstorAnswersThrow(answers); + + Long capacityIops = storagePool.getCapacityIops(); + if (capacityIops != null) + { + long vcIops = currentQosIops != null ? currentQosIops * -1 : 0; + long vMaxIops = maxIops != null ? maxIops : 0; + long newIops = vcIops + vMaxIops; + capacityIops -= newIops; + LOGGER.info("Current storagepool {} iops capacity: {}", storagePool, capacityIops); + storagePool.setCapacityIops(Math.max(0, capacityIops)); + primaryDataStoreDao.update(storagePool.getId(), storagePool); + } + } + } + + public static String createResource(VolumeInfo vol, StoragePoolVO storagePoolVO, + PrimaryDataStoreDao primaryDataStoreDao) { + return createResource(vol, storagePoolVO, primaryDataStoreDao, false); + } + + public static String createResource(VolumeInfo vol, StoragePoolVO storagePoolVO, + PrimaryDataStoreDao primaryDataStoreDao, boolean exactSize) { + DevelopersApi linstorApi = LinstorUtil.getLinstorAPI(storagePoolVO.getHostAddress()); + final String rscGrp = getRscGrp(storagePoolVO); + + final String rscName = LinstorUtil.RSC_PREFIX + vol.getUuid(); + createResourceBase( + rscName, vol.getSize(), vol.getName(), vol.getAttachedVmName(), vol.getPassphraseId(), vol.getPassphrase(), + linstorApi, rscGrp, storagePoolVO.getId(), false, exactSize); + + try + { + applyQoSSettings(primaryDataStoreDao, storagePoolVO, linstorApi, rscName, vol.getMaxIops()); + + return LinstorUtil.getDevicePath(linstorApi, rscName); + } catch (ApiException apiEx) + { + LOGGER.error("Linstor: ApiEx - " + apiEx.getMessage()); + throw new CloudRuntimeException(apiEx.getBestMessage(), apiEx); + } + } } diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/motion/LinstorDataMotionStrategy.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/motion/LinstorDataMotionStrategy.java new file mode 100644 index 00000000000..cab2820f09a --- /dev/null +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/motion/LinstorDataMotionStrategy.java @@ -0,0 +1,437 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.cloudstack.storage.motion; + +import com.linbit.linstor.api.ApiException; +import com.linbit.linstor.api.DevelopersApi; +import com.linbit.linstor.api.model.ApiCallRcList; +import com.linbit.linstor.api.model.ResourceDefinition; +import com.linbit.linstor.api.model.ResourceDefinitionModify; + +import javax.inject.Inject; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.cloud.agent.AgentManager; +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.MigrateAnswer; +import com.cloud.agent.api.MigrateCommand; +import com.cloud.agent.api.PrepareForMigrationCommand; +import com.cloud.agent.api.to.DataObjectType; +import com.cloud.agent.api.to.VirtualMachineTO; +import com.cloud.exception.AgentUnavailableException; +import com.cloud.exception.OperationTimedoutException; +import com.cloud.host.Host; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.storage.Storage; +import com.cloud.storage.StorageManager; +import com.cloud.storage.Volume; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.GuestOSCategoryDao; +import com.cloud.storage.dao.GuestOSDao; +import com.cloud.storage.dao.SnapshotDao; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.dao.VMInstanceDao; +import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; +import org.apache.cloudstack.engine.subsystem.api.storage.DataMotionStrategy; +import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; +import org.apache.cloudstack.engine.subsystem.api.storage.StrategyPriority; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService; +import org.apache.cloudstack.framework.async.AsyncCallFuture; +import org.apache.cloudstack.framework.async.AsyncCompletionCallback; +import org.apache.cloudstack.storage.command.CopyCmdAnswer; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.storage.datastore.util.LinstorUtil; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.stereotype.Component; + + +/** + * Current state: + * just changing the resource-group on same storage pool resource-group is not really good enough. + * Linstor lacks currently of a good way to move resources to another resource-group and respecting + * every auto-filter setting. + * Also linstor clone would simply set the new resource-group without any adjustments of storage pools or + * auto-select resource placement. + * So currently, we will create a new resource in the wanted primary storage and let qemu copy the data into the + * devices. + */ + +@Component +public class LinstorDataMotionStrategy implements DataMotionStrategy { + protected Logger logger = LogManager.getLogger(getClass()); + + @Inject + private SnapshotDataStoreDao _snapshotStoreDao; + @Inject + private PrimaryDataStoreDao _storagePool; + @Inject + private VolumeDao _volumeDao; + @Inject + private VolumeDataFactory _volumeDataFactory; + @Inject + private VMInstanceDao _vmDao; + @Inject + private GuestOSDao _guestOsDao; + @Inject + private VolumeService _volumeService; + @Inject + private GuestOSCategoryDao _guestOsCategoryDao; + @Inject + private SnapshotDao _snapshotDao; + @Inject + private AgentManager _agentManager; + @Inject + private PrimaryDataStoreDao _storagePoolDao; + + @Override + public StrategyPriority canHandle(DataObject srcData, DataObject dstData) { + DataObjectType srcType = srcData.getType(); + DataObjectType dstType = dstData.getType(); + logger.debug("canHandle: {} -> {}", srcType, dstType); + return StrategyPriority.CANT_HANDLE; + } + + @Override + public void copyAsync(DataObject srcData, DataObject destData, Host destHost, + AsyncCompletionCallback callback) { + throw new CloudRuntimeException("not implemented"); + } + + private boolean isDestinationLinstorPrimaryStorage(Map volumeMap) { + if (MapUtils.isNotEmpty(volumeMap)) { + for (DataStore dataStore : volumeMap.values()) { + StoragePoolVO storagePoolVO = _storagePool.findById(dataStore.getId()); + if (storagePoolVO == null + || !storagePoolVO.getStorageProviderName().equals(LinstorUtil.PROVIDER_NAME)) { + return false; + } + } + } else { + return false; + } + return true; + } + + @Override + public StrategyPriority canHandle(Map volumeMap, Host srcHost, Host destHost) { + logger.debug("canHandle -- {}: {} -> {}", volumeMap, srcHost, destHost); + if (srcHost.getId() != destHost.getId() && isDestinationLinstorPrimaryStorage(volumeMap)) { + return StrategyPriority.HIGHEST; + } + return StrategyPriority.CANT_HANDLE; + } + + private VolumeVO createNewVolumeVO(Volume volume, StoragePoolVO storagePoolVO) { + VolumeVO newVol = new VolumeVO(volume); + newVol.setInstanceId(null); + newVol.setChainInfo(null); + newVol.setPath(newVol.getUuid()); + newVol.setFolder(null); + newVol.setPodId(storagePoolVO.getPodId()); + newVol.setPoolId(storagePoolVO.getId()); + newVol.setLastPoolId(volume.getPoolId()); + + return _volumeDao.persist(newVol); + } + + private void removeExactSizeProperty(VolumeInfo volumeInfo) { + StoragePoolVO destStoragePool = _storagePool.findById(volumeInfo.getDataStore().getId()); + DevelopersApi api = LinstorUtil.getLinstorAPI(destStoragePool.getHostAddress()); + + ResourceDefinitionModify rdm = new ResourceDefinitionModify(); + rdm.setDeleteProps(Collections.singletonList(LinstorUtil.LIN_PROP_DRBDOPT_EXACT_SIZE)); + try { + String rscName = LinstorUtil.RSC_PREFIX + volumeInfo.getPath(); + ApiCallRcList answers = api.resourceDefinitionModify(rscName, rdm); + LinstorUtil.checkLinstorAnswersThrow(answers); + } catch (ApiException apiEx) { + logger.error("Linstor: ApiEx - {}", apiEx.getMessage()); + throw new CloudRuntimeException(apiEx.getBestMessage(), apiEx); + } + } + + private void handlePostMigration(boolean success, Map srcVolumeInfoToDestVolumeInfo, + VirtualMachineTO vmTO, Host destHost) { + if (!success) { + try { + PrepareForMigrationCommand pfmc = new PrepareForMigrationCommand(vmTO); + + pfmc.setRollback(true); + + Answer pfma = _agentManager.send(destHost.getId(), pfmc); + + if (pfma == null || !pfma.getResult()) { + String details = pfma != null ? pfma.getDetails() : "null answer returned"; + String msg = "Unable to rollback prepare for migration due to the following: " + details; + + throw new AgentUnavailableException(msg, destHost.getId()); + } + } catch (Exception e) { + logger.debug("Failed to disconnect one or more (original) dest volumes", e); + } + } + + for (Map.Entry entry : srcVolumeInfoToDestVolumeInfo.entrySet()) { + VolumeInfo srcVolumeInfo = entry.getKey(); + VolumeInfo destVolumeInfo = entry.getValue(); + + if (success) { + srcVolumeInfo.processEvent(ObjectInDataStoreStateMachine.Event.OperationSucceeded); + destVolumeInfo.processEvent(ObjectInDataStoreStateMachine.Event.OperationSucceeded); + + _volumeDao.updateUuid(srcVolumeInfo.getId(), destVolumeInfo.getId()); + + VolumeVO volumeVO = _volumeDao.findById(destVolumeInfo.getId()); + + volumeVO.setFormat(Storage.ImageFormat.QCOW2); + + _volumeDao.update(volumeVO.getId(), volumeVO); + + // remove exact size property + removeExactSizeProperty(destVolumeInfo); + + try { + _volumeService.destroyVolume(srcVolumeInfo.getId()); + + srcVolumeInfo = _volumeDataFactory.getVolume(srcVolumeInfo.getId()); + + AsyncCallFuture destroyFuture = + _volumeService.expungeVolumeAsync(srcVolumeInfo); + + if (destroyFuture.get().isFailed()) { + logger.debug("Failed to clean up source volume on storage"); + } + } catch (Exception e) { + logger.debug("Failed to clean up source volume on storage", e); + } + + // Update the volume ID for snapshots on secondary storage + if (!_snapshotDao.listByVolumeId(srcVolumeInfo.getId()).isEmpty()) { + _snapshotDao.updateVolumeIds(srcVolumeInfo.getId(), destVolumeInfo.getId()); + _snapshotStoreDao.updateVolumeIds(srcVolumeInfo.getId(), destVolumeInfo.getId()); + } + } else { + try { + _volumeService.revokeAccess(destVolumeInfo, destHost, destVolumeInfo.getDataStore()); + } catch (Exception e) { + logger.debug("Failed to revoke access from dest volume", e); + } + + destVolumeInfo.processEvent(ObjectInDataStoreStateMachine.Event.OperationFailed); + srcVolumeInfo.processEvent(ObjectInDataStoreStateMachine.Event.OperationFailed); + + try { + _volumeService.destroyVolume(destVolumeInfo.getId()); + + destVolumeInfo = _volumeDataFactory.getVolume(destVolumeInfo.getId()); + + AsyncCallFuture destroyFuture = + _volumeService.expungeVolumeAsync(destVolumeInfo); + + if (destroyFuture.get().isFailed()) { + logger.debug("Failed to clean up dest volume on storage"); + } + } catch (Exception e) { + logger.debug("Failed to clean up dest volume on storage", e); + } + } + } + } + + /** + * Determines whether the destination volume should have the DRBD exact-size property set + * during migration. + * + *

This method queries the Linstor API to check if the source volume's resource definition + * has the exact-size DRBD option enabled. The exact-size property ensures that DRBD uses + * the precise volume size rather than rounding, which is important for maintaining size + * consistency during migrations.

+ * + * @param srcVolumeInfo the source volume information to check + * @return {@code true} if the exact-size property should be set on the destination volume, + * which occurs when the source volume has this property enabled, or when the + * property cannot be determined (defaults to {@code true} for safety); + * {@code false} only when the source is confirmed to not have the exact-size property + */ + private boolean needsExactSizeProp(VolumeInfo srcVolumeInfo) { + StoragePoolVO srcStoragePool = _storagePool.findById(srcVolumeInfo.getDataStore().getId()); + if (srcStoragePool.getPoolType() == Storage.StoragePoolType.Linstor) { + DevelopersApi api = LinstorUtil.getLinstorAPI(srcStoragePool.getHostAddress()); + + String rscName = LinstorUtil.RSC_PREFIX + srcVolumeInfo.getPath(); + try { + List rscDfns = api.resourceDefinitionList( + Collections.singletonList(rscName), + false, + Collections.emptyList(), + null, + null); + if (!CollectionUtils.isEmpty(rscDfns)) { + ResourceDefinition srcRsc = rscDfns.get(0); + String exactSizeProp = srcRsc.getProps().get(LinstorUtil.LIN_PROP_DRBDOPT_EXACT_SIZE); + return "true".equalsIgnoreCase(exactSizeProp); + } else { + logger.warn("Unknown resource {} on {}", rscName, srcStoragePool.getHostAddress()); + } + } catch (ApiException apiEx) { + logger.error("Unable to fetch resource definition {}: {}", rscName, apiEx.getBestMessage()); + } + } + return true; + } + + @Override + public void copyAsync(Map volumeDataStoreMap, VirtualMachineTO vmTO, Host srcHost, + Host destHost, AsyncCompletionCallback callback) { + + if (srcHost.getHypervisorType() != Hypervisor.HypervisorType.KVM) { + throw new CloudRuntimeException( + String.format("Invalid hypervisor type [%s]. Only KVM supported", srcHost.getHypervisorType())); + } + + String errMsg = null; + VMInstanceVO vmInstance = _vmDao.findById(vmTO.getId()); + vmTO.setState(vmInstance.getState()); + List migrateDiskInfoList = new ArrayList<>(); + + Map migrateStorage = new HashMap<>(); + Map srcVolumeInfoToDestVolumeInfo = new HashMap<>(); + + try { + for (Map.Entry entry : volumeDataStoreMap.entrySet()) { + VolumeInfo srcVolumeInfo = entry.getKey(); + DataStore destDataStore = entry.getValue(); + VolumeVO srcVolume = _volumeDao.findById(srcVolumeInfo.getId()); + StoragePoolVO destStoragePool = _storagePool.findById(destDataStore.getId()); + + if (srcVolumeInfo.getPassphraseId() != null) { + throw new CloudRuntimeException( + String.format("Cannot live migrate encrypted volume: %s", srcVolumeInfo.getVolume())); + } + + VolumeVO destVolume = createNewVolumeVO(srcVolume, destStoragePool); + + VolumeInfo destVolumeInfo = _volumeDataFactory.getVolume(destVolume.getId(), destDataStore); + + destVolumeInfo.processEvent(ObjectInDataStoreStateMachine.Event.MigrationCopyRequested); + destVolumeInfo.processEvent(ObjectInDataStoreStateMachine.Event.MigrationCopySucceeded); + destVolumeInfo.processEvent(ObjectInDataStoreStateMachine.Event.MigrationRequested); + + boolean exactSize = needsExactSizeProp(srcVolumeInfo); + + String devPath = LinstorUtil.createResource( + destVolumeInfo, destStoragePool, _storagePoolDao, exactSize); + + _volumeDao.update(destVolume.getId(), destVolume); + destVolume = _volumeDao.findById(destVolume.getId()); + + destVolumeInfo = _volumeDataFactory.getVolume(destVolume.getId(), destDataStore); + + MigrateCommand.MigrateDiskInfo migrateDiskInfo = new MigrateCommand.MigrateDiskInfo( + srcVolumeInfo.getPath(), + MigrateCommand.MigrateDiskInfo.DiskType.BLOCK, + MigrateCommand.MigrateDiskInfo.DriverType.RAW, + MigrateCommand.MigrateDiskInfo.Source.DEV, + devPath); + migrateDiskInfoList.add(migrateDiskInfo); + + migrateStorage.put(srcVolumeInfo.getPath(), migrateDiskInfo); + + srcVolumeInfoToDestVolumeInfo.put(srcVolumeInfo, destVolumeInfo); + } + + PrepareForMigrationCommand pfmc = new PrepareForMigrationCommand(vmTO); + try { + Answer pfma = _agentManager.send(destHost.getId(), pfmc); + + if (pfma == null || !pfma.getResult()) { + String details = pfma != null ? pfma.getDetails() : "null answer returned"; + errMsg = String.format("Unable to prepare for migration due to the following: %s", details); + + throw new AgentUnavailableException(errMsg, destHost.getId()); + } + } catch (final OperationTimedoutException e) { + errMsg = String.format("Operation timed out due to %s", e.getMessage()); + throw new AgentUnavailableException(errMsg, destHost.getId()); + } + + VMInstanceVO vm = _vmDao.findById(vmTO.getId()); + boolean isWindows = _guestOsCategoryDao.findById(_guestOsDao.findById(vm.getGuestOSId()).getCategoryId()) + .getName().equalsIgnoreCase("Windows"); + + MigrateCommand migrateCommand = new MigrateCommand(vmTO.getName(), + destHost.getPrivateIpAddress(), isWindows, vmTO, true); + migrateCommand.setWait(StorageManager.KvmStorageOnlineMigrationWait.value()); + migrateCommand.setMigrateStorage(migrateStorage); + migrateCommand.setMigrateStorageManaged(true); + migrateCommand.setNewVmCpuShares( + vmTO.getCpus() * ObjectUtils.defaultIfNull(vmTO.getMinSpeed(), vmTO.getSpeed())); + migrateCommand.setMigrateDiskInfoList(migrateDiskInfoList); + + boolean kvmAutoConvergence = StorageManager.KvmAutoConvergence.value(); + migrateCommand.setAutoConvergence(kvmAutoConvergence); + + MigrateAnswer migrateAnswer = (MigrateAnswer) _agentManager.send(srcHost.getId(), migrateCommand); + boolean success = migrateAnswer != null && migrateAnswer.getResult(); + + handlePostMigration(success, srcVolumeInfoToDestVolumeInfo, vmTO, destHost); + + if (migrateAnswer == null) { + throw new CloudRuntimeException("Unable to get an answer to the migrate command"); + } + + if (!migrateAnswer.getResult()) { + errMsg = migrateAnswer.getDetails(); + + throw new CloudRuntimeException(errMsg); + } + } catch (AgentUnavailableException | OperationTimedoutException | CloudRuntimeException ex) { + errMsg = String.format( + "Copy volume(s) of VM [%s] to storage(s) [%s] and VM to host [%s] failed in LinstorDataMotionStrategy.copyAsync. Error message: [%s].", + vmTO, srcHost, destHost, ex.getMessage()); + logger.error(errMsg, ex); + + throw new CloudRuntimeException(errMsg); + } finally { + CopyCmdAnswer copyCmdAnswer = new CopyCmdAnswer(errMsg); + + CopyCommandResult result = new CopyCommandResult(null, copyCmdAnswer); + result.setResult(errMsg); + callback.complete(result); + } + } +} diff --git a/plugins/storage/volume/linstor/src/main/resources/META-INF/cloudstack/storage-volume-linstor/spring-storage-volume-linstor-context.xml b/plugins/storage/volume/linstor/src/main/resources/META-INF/cloudstack/storage-volume-linstor/spring-storage-volume-linstor-context.xml index a900323ede5..88d1051c71e 100644 --- a/plugins/storage/volume/linstor/src/main/resources/META-INF/cloudstack/storage-volume-linstor/spring-storage-volume-linstor-context.xml +++ b/plugins/storage/volume/linstor/src/main/resources/META-INF/cloudstack/storage-volume-linstor/spring-storage-volume-linstor-context.xml @@ -33,4 +33,6 @@ class="org.apache.cloudstack.storage.snapshot.LinstorVMSnapshotStrategy" /> + diff --git a/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImplTest.java b/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImplTest.java index 75276739468..4653cfa358b 100644 --- a/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImplTest.java +++ b/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImplTest.java @@ -26,6 +26,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import org.apache.cloudstack.storage.datastore.util.LinstorUtil; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -75,13 +76,13 @@ public class LinstorPrimaryDataStoreDriverImplTest { when(api.resourceGroupList(Collections.singletonList("EncryptedGrp"), Collections.emptyList(), null, null)) .thenReturn(Collections.singletonList(encryptedGrp)); - List layers = linstorPrimaryDataStoreDriver.getEncryptedLayerList(api, "DfltRscGrp"); + List layers = LinstorUtil.getEncryptedLayerList(api, "DfltRscGrp"); Assert.assertEquals(Arrays.asList(LayerType.DRBD, LayerType.LUKS, LayerType.STORAGE), layers); - layers = linstorPrimaryDataStoreDriver.getEncryptedLayerList(api, "BcacheGrp"); + layers = LinstorUtil.getEncryptedLayerList(api, "BcacheGrp"); Assert.assertEquals(Arrays.asList(LayerType.DRBD, LayerType.BCACHE, LayerType.LUKS, LayerType.STORAGE), layers); - layers = linstorPrimaryDataStoreDriver.getEncryptedLayerList(api, "EncryptedGrp"); + layers = LinstorUtil.getEncryptedLayerList(api, "EncryptedGrp"); Assert.assertEquals(Arrays.asList(LayerType.DRBD, LayerType.LUKS, LayerType.STORAGE), layers); } } diff --git a/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/driver/SolidFirePrimaryDataStoreDriver.java b/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/driver/SolidFirePrimaryDataStoreDriver.java index 6cc76d99d9e..1e927e20168 100644 --- a/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/driver/SolidFirePrimaryDataStoreDriver.java +++ b/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/driver/SolidFirePrimaryDataStoreDriver.java @@ -433,7 +433,7 @@ public class SolidFirePrimaryDataStoreDriver implements PrimaryDataStoreDriver { public long getUsedIops(StoragePool storagePool) { long usedIops = 0; - List volumes = volumeDao.findByPoolId(storagePool.getId(), null); + List volumes = volumeDao.findNonDestroyedVolumesByPoolId(storagePool.getId(), null); if (volumes != null) { for (VolumeVO volume : volumes) { diff --git a/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/provider/SolidFireHostListener.java b/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/provider/SolidFireHostListener.java index 052191128f1..c961c926739 100644 --- a/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/provider/SolidFireHostListener.java +++ b/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/provider/SolidFireHostListener.java @@ -199,7 +199,7 @@ public class SolidFireHostListener implements HypervisorHostListener { List storagePaths = new ArrayList<>(); // If you do not pass in null for the second parameter, you only get back applicable ROOT disks. - List volumes = volumeDao.findByPoolId(storagePoolId, null); + List volumes = volumeDao.findNonDestroyedVolumesByPoolId(storagePoolId, null); if (volumes != null) { for (VolumeVO volume : volumes) { @@ -230,7 +230,7 @@ public class SolidFireHostListener implements HypervisorHostListener { StoragePoolVO storagePool = storagePoolDao.findById(storagePoolId); // If you do not pass in null for the second parameter, you only get back applicable ROOT disks. - List volumes = volumeDao.findByPoolId(storagePoolId, null); + List volumes = volumeDao.findNonDestroyedVolumesByPoolId(storagePoolId, null); if (volumes != null) { for (VolumeVO volume : volumes) { diff --git a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/driver/StorPoolPrimaryDataStoreDriver.java b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/driver/StorPoolPrimaryDataStoreDriver.java index 9dc94c20f11..f666711f744 100644 --- a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/driver/StorPoolPrimaryDataStoreDriver.java +++ b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/driver/StorPoolPrimaryDataStoreDriver.java @@ -1354,7 +1354,7 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver { return volumeStats; } } else { - List volumes = volumeDao.findByPoolId(storagePool.getId()); + List volumes = volumeDao.findNonDestroyedVolumesByPoolId(storagePool.getId()); for (VolumeVO volume : volumes) { if (volume.getPath() != null && volume.getPath().equals(volumeId)) { long size = volume.getSize(); diff --git a/plugins/user-authenticators/oauth2/src/main/java/org/apache/cloudstack/oauth2/api/command/VerifyOAuthCodeAndGetUserCmd.java b/plugins/user-authenticators/oauth2/src/main/java/org/apache/cloudstack/oauth2/api/command/VerifyOAuthCodeAndGetUserCmd.java index bd49f87d627..b3d2d335ba2 100644 --- a/plugins/user-authenticators/oauth2/src/main/java/org/apache/cloudstack/oauth2/api/command/VerifyOAuthCodeAndGetUserCmd.java +++ b/plugins/user-authenticators/oauth2/src/main/java/org/apache/cloudstack/oauth2/api/command/VerifyOAuthCodeAndGetUserCmd.java @@ -20,8 +20,10 @@ import java.net.InetAddress; import java.util.List; import java.util.Map; -import com.cloud.api.response.ApiResponseSerializer; -import com.cloud.user.Account; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.servlet.http.HttpSession; + import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; @@ -37,13 +39,13 @@ import org.apache.cloudstack.oauth2.OAuth2AuthManager; import org.apache.cloudstack.oauth2.api.response.OauthProviderResponse; import org.apache.commons.lang.ArrayUtils; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import javax.servlet.http.HttpSession; +import com.cloud.api.response.ApiResponseSerializer; +import com.cloud.user.Account; @APICommand(name = "verifyOAuthCodeAndGetUser", description = "Verify the OAuth Code and fetch the corresponding user from provider", responseObject = OauthProviderResponse.class, entityType = {}, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, - authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}, since = "4.19.0") + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}, since = "4.19.0", + httpMethod = "GET") public class VerifyOAuthCodeAndGetUserCmd extends BaseListCmd implements APIAuthenticator { ///////////////////////////////////////////////////// diff --git a/plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/saml/SAML2AuthManager.java b/plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/saml/SAML2AuthManager.java index 523f694d80b..618536a71f6 100644 --- a/plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/saml/SAML2AuthManager.java +++ b/plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/saml/SAML2AuthManager.java @@ -82,6 +82,9 @@ public interface SAML2AuthManager extends PluggableAPIAuthenticator, PluggableSe ConfigKey SAMLRequirePasswordLogin = new ConfigKey("Advanced", Boolean.class, "saml2.require.password", "true", "When enabled SAML2 will validate that the SAML login was performed with a password. If disabled, other forms of authentication are allowed (two-factor, certificate, etc) on the SAML Authentication Provider", true); + ConfigKey EnableLoginAfterSAMLDisable = new ConfigKey<>("Advanced", Boolean.class, "enable.login.with.disabled.saml", "false", "When enabled, if SAML SSO is disabled, enables user to login with user and password, otherwise a user with SAML SSO disabled cannot login", true); + + SAMLProviderMetadata getSPMetadata(); SAMLProviderMetadata getIdPMetadata(String entityId); diff --git a/plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/saml/SAML2AuthManagerImpl.java b/plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/saml/SAML2AuthManagerImpl.java index 93b7bc5be93..9f8101b867d 100644 --- a/plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/saml/SAML2AuthManagerImpl.java +++ b/plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/saml/SAML2AuthManagerImpl.java @@ -451,8 +451,13 @@ public class SAML2AuthManagerImpl extends AdapterBase implements SAML2AuthManage user.setExternalEntity(entityId); user.setSource(User.Source.SAML2); } else { + boolean enableLoginAfterSAMLDisable = SAML2AuthManager.EnableLoginAfterSAMLDisable.value(); if (user.getSource().equals(User.Source.SAML2)) { - user.setSource(User.Source.SAML2DISABLED); + if(enableLoginAfterSAMLDisable) { + user.setSource(User.Source.UNKNOWN); + } else { + user.setSource(User.Source.SAML2DISABLED); + } } else { return false; } @@ -541,6 +546,6 @@ public class SAML2AuthManagerImpl extends AdapterBase implements SAML2AuthManage SAMLCloudStackRedirectionUrl, SAMLUserAttributeName, SAMLIdentityProviderMetadataURL, SAMLDefaultIdentityProviderId, SAMLSignatureAlgorithm, SAMLAppendDomainSuffix, SAMLTimeout, SAMLCheckSignature, - SAMLForceAuthn, SAMLUserSessionKeyPathAttribute, SAMLRequirePasswordLogin}; + SAMLForceAuthn, SAMLUserSessionKeyPathAttribute, SAMLRequirePasswordLogin, EnableLoginAfterSAMLDisable}; } } diff --git a/pom.xml b/pom.xml index d5b5ba121c9..883e7a4e4db 100644 --- a/pom.xml +++ b/pom.xml @@ -470,8 +470,8 @@ ${cs.reload4j.version} - mysql - mysql-connector-java + com.mysql + mysql-connector-j ${cs.mysql.version} test @@ -486,12 +486,6 @@ - - com.mysql - mysql-connector-j - ${cs.mysql.version} - test - net.sf.ehcache ehcache-core diff --git a/scripts/storage/secondary/cloud-install-sys-tmplt b/scripts/storage/secondary/cloud-install-sys-tmplt index ad976c502c6..fc09dc968ff 100755 --- a/scripts/storage/secondary/cloud-install-sys-tmplt +++ b/scripts/storage/secondary/cloud-install-sys-tmplt @@ -44,6 +44,7 @@ failed() { } #set -x +umask 0022 # ensure we have the proper permissions even on hardened deployments mflag= fflag= ext="vhd" diff --git a/scripts/storage/secondary/setup-sysvm-tmplt b/scripts/storage/secondary/setup-sysvm-tmplt index 06f0586fe34..63006cc4e4c 100755 --- a/scripts/storage/secondary/setup-sysvm-tmplt +++ b/scripts/storage/secondary/setup-sysvm-tmplt @@ -19,6 +19,7 @@ # Usage: e.g. failed $? "this is an error" set -x +umask 0022 # ensure we have the proper permissions even on hardened deployments failed() { local returnval=$1 diff --git a/scripts/vm/hypervisor/external/provisioner/provisioner.sh b/scripts/vm/hypervisor/external/provisioner/provisioner.sh index f067d892f1f..c92ac36f466 100755 --- a/scripts/vm/hypervisor/external/provisioner/provisioner.sh +++ b/scripts/vm/hypervisor/external/provisioner/provisioner.sh @@ -99,6 +99,14 @@ status() { echo '{"status": "success", "power_state": "poweron"}' } +statuses() { + parse_json "$1" || exit 1 + # This external system can not return an output like the following: + # {"status":"success","power_state":{"i-3-23-VM":"poweroff","i-2-25-VM":"poweron"}} + # CloudStack can fallback to retrieving the power state of the single VM using the "status" action + echo '{"status": "error", "message": "Not supported"}' +} + get_console() { parse_json "$1" || exit 1 local response @@ -145,6 +153,9 @@ case $action in status) status "$parameters" ;; + statuses) + statuses "$parameters" + ;; getconsole) get_console "$parameters" ;; diff --git a/scripts/vm/hypervisor/xenserver/xenserver84/vmops b/scripts/vm/hypervisor/xenserver/xenserver84/vmops index cf6e6325d68..76d4571ef18 100755 --- a/scripts/vm/hypervisor/xenserver/xenserver84/vmops +++ b/scripts/vm/hypervisor/xenserver/xenserver84/vmops @@ -1587,6 +1587,43 @@ def network_rules(session, args): except: logging.exception("Failed to network rule!") +@echo +def create_vtpm(session, args): + util.SMlog("create_vtpm called with args: %s" % str(args)) + + try: + vm_uuid = args.get('vm_uuid') + if not vm_uuid: + return "ERROR: vm_uuid parameter is required" + + # Check if vTPM already exists for this VM + cmd = ['xe', 'vtpm-list', 'vm-uuid=' + vm_uuid, '--minimal'] + result = util.pread2(cmd) + existing_vtpms = result.strip() + + if existing_vtpms: + util.SMlog("vTPM already exists for VM %s: %s" % (vm_uuid, existing_vtpms)) + return existing_vtpms.split(',')[0] + + cmd = ['xe', 'vtpm-create', 'vm-uuid=' + vm_uuid] + result = util.pread2(cmd) + vtpm_uuid = result.strip() + + if vtpm_uuid: + util.SMlog("Successfully created vTPM %s for VM %s" % (vtpm_uuid, vm_uuid)) + return vtpm_uuid + else: + return "ERROR: Failed to create vTPM, empty result" + + except CommandException as e: + error_msg = "xe command failed: %s" % str(e) + util.SMlog("ERROR: %s" % error_msg) + return "ERROR: " + error_msg + except Exception as e: + error_msg = str(e) + util.SMlog("ERROR: %s" % error_msg) + return "ERROR: " + error_msg + if __name__ == "__main__": XenAPIPlugin.dispatch({"pingtest": pingtest, "setup_iscsi":setup_iscsi, "preparemigration": preparemigration, @@ -1604,4 +1641,5 @@ if __name__ == "__main__": "createFileInDomr":createFileInDomr, "kill_copy_process":kill_copy_process, "secureCopyToHost":secureCopyToHost, - "runPatchScriptInDomr": runPatchScriptInDomr}) + "runPatchScriptInDomr": runPatchScriptInDomr, + "create_vtpm": create_vtpm}) diff --git a/server/src/main/java/com/cloud/alert/AlertManagerImpl.java b/server/src/main/java/com/cloud/alert/AlertManagerImpl.java index 308c4443cb8..27b445ba376 100644 --- a/server/src/main/java/com/cloud/alert/AlertManagerImpl.java +++ b/server/src/main/java/com/cloud/alert/AlertManagerImpl.java @@ -19,7 +19,6 @@ package com.cloud.alert; import java.io.UnsupportedEncodingException; import java.text.DecimalFormat; import java.util.ArrayList; -import java.util.Arrays; import java.util.Date; import java.util.HashMap; import java.util.HashSet; @@ -37,15 +36,12 @@ import javax.inject.Inject; import javax.mail.MessagingException; import javax.naming.ConfigurationException; -import com.cloud.dc.DataCenter; -import com.cloud.dc.Pod; -import com.cloud.org.Cluster; - import org.apache.cloudstack.backup.BackupManager; import org.apache.cloudstack.framework.config.ConfigDepot; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.framework.messagebus.MessageBus; import org.apache.cloudstack.managed.context.ManagedContextTimerTask; import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; @@ -54,6 +50,7 @@ import org.apache.cloudstack.utils.mailing.MailAddress; import org.apache.cloudstack.utils.mailing.SMTPMailProperties; import org.apache.cloudstack.utils.mailing.SMTPMailSender; import org.apache.commons.lang3.ArrayUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.math.NumberUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -70,9 +67,11 @@ import com.cloud.capacity.dao.CapacityDaoImpl.SummedCapacity; import com.cloud.configuration.Config; import com.cloud.configuration.ConfigurationManager; import com.cloud.dc.ClusterVO; +import com.cloud.dc.DataCenter; import com.cloud.dc.DataCenter.NetworkType; import com.cloud.dc.DataCenterVO; import com.cloud.dc.HostPodVO; +import com.cloud.dc.Pod; import com.cloud.dc.Vlan.VlanType; import com.cloud.dc.dao.ClusterDao; import com.cloud.dc.dao.DataCenterDao; @@ -86,10 +85,12 @@ import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; import com.cloud.network.Ipv6Service; import com.cloud.network.dao.IPAddressDao; +import com.cloud.org.Cluster; import com.cloud.org.Grouping.AllocationState; import com.cloud.resource.ResourceManager; import com.cloud.storage.StorageManager; import com.cloud.utils.Pair; +import com.cloud.utils.Ternary; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.concurrency.NamedThreadFactory; import com.cloud.utils.db.SearchCriteria; @@ -100,21 +101,6 @@ import com.cloud.utils.db.TransactionStatus; public class AlertManagerImpl extends ManagerBase implements AlertManager, Configurable { protected Logger logger = LogManager.getLogger(AlertManagerImpl.class.getName()); - public static final List ALERTS = Arrays.asList(AlertType.ALERT_TYPE_HOST - , AlertType.ALERT_TYPE_USERVM - , AlertType.ALERT_TYPE_DOMAIN_ROUTER - , AlertType.ALERT_TYPE_CONSOLE_PROXY - , AlertType.ALERT_TYPE_SSVM - , AlertType.ALERT_TYPE_STORAGE_MISC - , AlertType.ALERT_TYPE_MANAGEMENT_NODE - , AlertType.ALERT_TYPE_RESOURCE_LIMIT_EXCEEDED - , AlertType.ALERT_TYPE_UPLOAD_FAILED - , AlertType.ALERT_TYPE_OOBM_AUTH_ERROR - , AlertType.ALERT_TYPE_HA_ACTION - , AlertType.ALERT_TYPE_CA_CERT - , AlertType.ALERT_TYPE_EXTENSION_PATH_NOT_READY - , AlertType.ALERT_TYPE_VPN_GATEWAY_OBSOLETE_PARAMETERS); - private static final long INITIAL_CAPACITY_CHECK_DELAY = 30L * 1000L; // Thirty seconds expressed in milliseconds. private static final DecimalFormat DfPct = new DecimalFormat("###.##"); @@ -156,6 +142,8 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager, Confi Ipv6Service ipv6Service; @Inject HostDao hostDao; + @Inject + MessageBus messageBus; private Timer _timer = null; private long _capacityCheckPeriod = 60L * 60L * 1000L; // One hour by default. @@ -175,6 +163,8 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager, Confi protected String[] recipients = null; protected String senderAddress = null; + private final List allowedRepetitiveAlertTypeNames = new ArrayList<>(); + public AlertManagerImpl() { _executor = Executors.newCachedThreadPool(new NamedThreadFactory("Email-Alerts-Sender")); } @@ -254,12 +244,32 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager, Confi _capacityCheckPeriod = Long.parseLong(Config.CapacityCheckPeriod.getDefaultValue()); } } + initMessageBusListener(); + setupRepetitiveAlertTypes(); _timer = new Timer("CapacityChecker"); return true; } + protected void setupRepetitiveAlertTypes() { + allowedRepetitiveAlertTypeNames.clear(); + String allowedRepetitiveAlertsStr = AllowedRepetitiveAlertTypes.value(); + logger.trace("Allowed repetitive alert types specified by {}: {} ", AllowedRepetitiveAlertTypes.key(), + allowedRepetitiveAlertsStr); + if (StringUtils.isBlank(allowedRepetitiveAlertsStr)) { + return; + } + String[] allowedRepetitiveAlertTypesArray = allowedRepetitiveAlertsStr.split(","); + for (String allowedTypeName : allowedRepetitiveAlertTypesArray) { + if (StringUtils.isBlank(allowedTypeName)) { + continue; + } + allowedRepetitiveAlertTypeNames.add(allowedTypeName.toLowerCase()); + } + logger.trace("{} alert types specified for repetitive alerts", allowedRepetitiveAlertTypeNames.size()); + } + @Override public boolean start() { _timer.schedule(new CapacityChecker(), INITIAL_CAPACITY_CHECK_DELAY, _capacityCheckPeriod); @@ -850,11 +860,11 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager, Confi @Nullable private AlertVO getAlertForTrivialAlertType(AlertType alertType, long dataCenterId, Long podId, Long clusterId) { - AlertVO alert = null; - if (!ALERTS.contains(alertType)) { - alert = _alertDao.getLastAlert(alertType.getType(), dataCenterId, podId, clusterId); + if (alertType.isRepetitionAllowed() || (StringUtils.isNotBlank(alertType.getName()) && + allowedRepetitiveAlertTypeNames.contains(alertType.getName().toLowerCase()))) { + return null; } - return alert; + return _alertDao.getLastAlert(alertType.getType(), dataCenterId, podId, clusterId); } protected void sendMessage(SMTPMailProperties mailProps) { @@ -883,7 +893,7 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager, Confi @Override public ConfigKey[] getConfigKeys() { return new ConfigKey[] {CPUCapacityThreshold, MemoryCapacityThreshold, StorageAllocatedCapacityThreshold, StorageCapacityThreshold, AlertSmtpEnabledSecurityProtocols, - AlertSmtpUseStartTLS, Ipv6SubnetCapacityThreshold, AlertSmtpUseAuth}; + AlertSmtpUseStartTLS, Ipv6SubnetCapacityThreshold, AlertSmtpUseAuth, AllowedRepetitiveAlertTypes}; } @Override @@ -897,4 +907,16 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager, Confi return false; } } + + @SuppressWarnings("unchecked") + protected void initMessageBusListener() { + messageBus.subscribe(EventTypes.EVENT_CONFIGURATION_VALUE_EDIT, (senderAddress, subject, args) -> { + Ternary updatedSetting = (Ternary) args; + String updatedSettingName = updatedSetting.first(); + if (!AllowedRepetitiveAlertTypes.key().equals(updatedSettingName)) { + return; + } + setupRepetitiveAlertTypes(); + }); + } } diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index ce794cf5388..655f5acb46e 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -39,19 +39,6 @@ import java.util.stream.Collectors; import javax.inject.Inject; -import com.cloud.bgp.ASNumber; -import com.cloud.bgp.ASNumberRange; -import com.cloud.configuration.ConfigurationService; -import com.cloud.dc.ASNumberRangeVO; -import com.cloud.dc.ASNumberVO; -import com.cloud.dc.VlanDetailsVO; -import com.cloud.dc.dao.ASNumberDao; -import com.cloud.dc.dao.ASNumberRangeDao; -import com.cloud.dc.dao.VlanDetailsDao; -import com.cloud.hypervisor.Hypervisor; -import com.cloud.network.vpc.VpcGateway; -import com.cloud.network.vpn.Site2SiteVpnManager; -import com.cloud.storage.BucketVO; import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.affinity.AffinityGroup; @@ -277,14 +264,19 @@ import com.cloud.api.query.vo.UserVmJoinVO; import com.cloud.api.query.vo.VolumeJoinVO; import com.cloud.api.query.vo.VpcOfferingJoinVO; import com.cloud.api.response.ApiResponseSerializer; +import com.cloud.bgp.ASNumber; +import com.cloud.bgp.ASNumberRange; import com.cloud.capacity.Capacity; import com.cloud.capacity.CapacityVO; import com.cloud.capacity.dao.CapacityDaoImpl.SummedCapacity; import com.cloud.configuration.ConfigurationManager; +import com.cloud.configuration.ConfigurationService; import com.cloud.configuration.Resource.ResourceOwnerType; import com.cloud.configuration.Resource.ResourceType; import com.cloud.configuration.ResourceCount; import com.cloud.configuration.ResourceLimit; +import com.cloud.dc.ASNumberRangeVO; +import com.cloud.dc.ASNumberVO; import com.cloud.dc.ClusterDetailsDao; import com.cloud.dc.ClusterVO; import com.cloud.dc.DataCenter; @@ -295,7 +287,11 @@ import com.cloud.dc.Pod; import com.cloud.dc.StorageNetworkIpRange; import com.cloud.dc.Vlan; import com.cloud.dc.Vlan.VlanType; +import com.cloud.dc.VlanDetailsVO; import com.cloud.dc.VlanVO; +import com.cloud.dc.dao.ASNumberDao; +import com.cloud.dc.dao.ASNumberRangeDao; +import com.cloud.dc.dao.VlanDetailsDao; import com.cloud.domain.Domain; import com.cloud.domain.DomainVO; import com.cloud.event.Event; @@ -304,6 +300,7 @@ import com.cloud.exception.PermissionDeniedException; import com.cloud.host.ControlState; import com.cloud.host.Host; import com.cloud.host.HostVO; +import com.cloud.hypervisor.Hypervisor; import com.cloud.hypervisor.HypervisorCapabilities; import com.cloud.network.GuestVlan; import com.cloud.network.GuestVlanRange; @@ -367,9 +364,11 @@ import com.cloud.network.vpc.NetworkACLItem; import com.cloud.network.vpc.PrivateGateway; import com.cloud.network.vpc.StaticRoute; import com.cloud.network.vpc.Vpc; +import com.cloud.network.vpc.VpcGateway; import com.cloud.network.vpc.VpcOffering; import com.cloud.network.vpc.VpcVO; import com.cloud.network.vpc.dao.VpcOfferingDao; +import com.cloud.network.vpn.Site2SiteVpnManager; import com.cloud.offering.DiskOffering; import com.cloud.offering.NetworkOffering; import com.cloud.offering.NetworkOffering.Detail; @@ -388,6 +387,7 @@ import com.cloud.server.ResourceIconManager; import com.cloud.server.ResourceTag; import com.cloud.server.ResourceTag.ResourceObjectType; import com.cloud.service.ServiceOfferingVO; +import com.cloud.storage.BucketVO; import com.cloud.storage.DataStoreRole; import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.GuestOS; @@ -586,6 +586,7 @@ public class ApiResponseHelper implements ResponseGenerator { if (domain.getChildCount() > 0) { domainResponse.setHasChild(true); } + populateDomainTags(domain.getUuid(), domainResponse); domainResponse.setObjectName("domain"); return domainResponse; } @@ -1883,6 +1884,8 @@ public class ApiResponseHelper implements ResponseGenerator { vmResponse.setPublicNetmask(singleNicProfile.getIPv4Netmask()); vmResponse.setGateway(singleNicProfile.getIPv4Gateway()); } + } else if (network.getTrafficType() == TrafficType.Storage) { + vmResponse.setStorageIp(singleNicProfile.getIPv4Address()); } } } @@ -3048,6 +3051,20 @@ public class ApiResponseHelper implements ResponseGenerator { response.setDomainPath(getPrettyDomainPath(object.getDomainPath())); } + public static void populateDomainTags(String domainUuid, DomainResponse domainResponse) { + List tags = ApiDBUtils.listResourceTagViewByResourceUUID(domainUuid, + ResourceTag.ResourceObjectType.Domain); + if (CollectionUtils.isEmpty(tags)) { + return; + } + Set tagResponses = new HashSet<>(); + for (ResourceTagJoinVO tag : tags) { + ResourceTagResponse tagResponse = ApiDBUtils.newResourceTagResponse(tag, true); + tagResponses.add(tagResponse); + } + domainResponse.setTags(tagResponses); + } + private void populateAccount(ControlledEntityResponse response, long accountId) { Account account = ApiDBUtils.findAccountById(accountId); if (account == null) { @@ -4462,7 +4479,7 @@ public class ApiResponseHelper implements ResponseGenerator { } else if (usageRecord.getUsageType() == UsageTypes.BACKUP) { resourceType = ResourceObjectType.Backup; final StringBuilder builder = new StringBuilder(); - builder.append("Backup usage of size ").append(usageRecord.getUsageDisplay()); + builder.append("Backup usage"); if (vmInstance != null) { resourceId = vmInstance.getId(); usageRecResponse.setResourceName(vmInstance.getInstanceName()); @@ -4475,9 +4492,12 @@ public class ApiResponseHelper implements ResponseGenerator { .append(" (").append(backupOffering.getUuid()).append(", user ad-hoc/scheduled backup allowed: ") .append(backupOffering.isUserDrivenBackupAllowed()).append(")"); } - } + builder.append(" with size ").append(toHumanReadableSize(usageRecord.getSize())); + builder.append(" and with virtual size ").append(toHumanReadableSize(usageRecord.getVirtualSize())); usageRecResponse.setDescription(builder.toString()); + usageRecResponse.setSize(usageRecord.getSize()); + usageRecResponse.setVirtualSize(usageRecord.getVirtualSize()); } else if (usageRecord.getUsageType() == UsageTypes.VM_SNAPSHOT) { resourceType = ResourceObjectType.VMSnapshot; VMSnapshotVO vmSnapshotVO = null; diff --git a/server/src/main/java/com/cloud/api/ApiServlet.java b/server/src/main/java/com/cloud/api/ApiServlet.java index 21d09375812..158df224071 100644 --- a/server/src/main/java/com/cloud/api/ApiServlet.java +++ b/server/src/main/java/com/cloud/api/ApiServlet.java @@ -25,8 +25,8 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.regex.Pattern; import java.util.Set; +import java.util.regex.Pattern; import javax.inject.Inject; import javax.servlet.ServletConfig; @@ -52,10 +52,9 @@ import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.managed.context.ManagedContext; import org.apache.cloudstack.utils.consoleproxy.ConsoleAccessUtils; import org.apache.commons.collections.MapUtils; - -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.LogManager; import org.apache.commons.lang3.EnumUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.Nullable; import org.springframework.stereotype.Component; import org.springframework.web.context.support.SpringBeanAutowiringSupport; @@ -70,12 +69,12 @@ import com.cloud.user.AccountManagerImpl; import com.cloud.user.AccountService; import com.cloud.user.User; import com.cloud.user.UserAccount; - import com.cloud.utils.HttpUtils; -import com.cloud.utils.HttpUtils.ApiSessionKeySameSite; import com.cloud.utils.HttpUtils.ApiSessionKeyCheckOption; +import com.cloud.utils.HttpUtils.ApiSessionKeySameSite; import com.cloud.utils.StringUtils; import com.cloud.utils.db.EntityManager; +import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.net.NetUtils; @Component("apiServlet") @@ -84,9 +83,7 @@ public class ApiServlet extends HttpServlet { private static final Logger ACCESSLOGGER = LogManager.getLogger("apiserver." + ApiServlet.class.getName()); private static final String REPLACEMENT = "_"; private static final String LOGGER_REPLACEMENTS = "[\n\r\t]"; - private static final Pattern GET_REQUEST_COMMANDS = Pattern.compile("^(get|list|query|find)(\\w+)+$"); - private static final HashSet GET_REQUEST_COMMANDS_LIST = new HashSet<>(Set.of("isaccountallowedtocreateofferingswithtags", - "readyforshutdown", "cloudianisenabled", "quotabalance", "quotasummary", "quotatarifflist", "quotaisenabled", "quotastatement", "verifyoauthcodeandgetuser")); + public static final Pattern GET_REQUEST_COMMANDS = Pattern.compile("^(get|list|query|find)(\\w+)+$"); private static final HashSet POST_REQUESTS_TO_DISABLE_LOGGING = new HashSet<>(Set.of( "login", "oauthlogin", @@ -367,7 +364,7 @@ public class ApiServlet extends HttpServlet { } } - if (apiServer.isPostRequestsAndTimestampsEnforced() && !isStateChangingCommandUsingPOST(command, req.getMethod(), params)) { + if (apiServer.isPostRequestsAndTimestampsEnforced() && isStateChangingCommandNotUsingPOST(command, req.getMethod(), params)) { String errorText = String.format("State changing command %s needs to be sent using POST request", command); if (command.equalsIgnoreCase("updateConfiguration") && params.containsKey("name")) { errorText = String.format("Changes for configuration %s needs to be sent using POST request", params.get("name")[0]); @@ -485,13 +482,32 @@ public class ApiServlet extends HttpServlet { return verify2FA; } - private boolean isStateChangingCommandUsingPOST(String command, String method, Map params) { - if (command == null || (!GET_REQUEST_COMMANDS.matcher(command.toLowerCase()).matches() && !GET_REQUEST_COMMANDS_LIST.contains(command.toLowerCase()) - && !command.equalsIgnoreCase("updateConfiguration") && !method.equals("POST"))) { + protected boolean isStateChangingCommandNotUsingPOST(String command, String method, Map params) { + if (BaseCmd.HTTPMethod.POST.toString().equalsIgnoreCase(method)) { return false; } - return !command.equalsIgnoreCase("updateConfiguration") || method.equals("POST") || (params.containsKey("name") - && params.get("name")[0].toString().equalsIgnoreCase(ApiServer.EnforcePostRequestsAndTimestamps.key())); + if (command == null || method == null) { + return true; + } + String commandHttpMethod = null; + try { + Class cmdClass = apiServer.getCmdClass(command); + if (cmdClass != null) { + APICommand at = cmdClass.getAnnotation(APICommand.class); + if (at != null && org.apache.commons.lang3.StringUtils.isNotBlank(at.httpMethod())) { + commandHttpMethod = at.httpMethod(); + } + } + } catch (CloudRuntimeException e) { + LOGGER.trace("Command class not found for {}; falling back to pattern match", command, e); + } + if (BaseCmd.HTTPMethod.GET.toString().equalsIgnoreCase(commandHttpMethod) || + GET_REQUEST_COMMANDS.matcher(command.toLowerCase()).matches()) { + return false; + } + return !command.equalsIgnoreCase("updateConfiguration") || + !params.containsKey("name") || + !ApiServer.EnforcePostRequestsAndTimestamps.key().equalsIgnoreCase(params.get("name")[0].toString()); } protected boolean skip2FAcheckForAPIs(String command) { @@ -679,38 +695,34 @@ public class ApiServlet extends HttpServlet { } return false; } - boolean doUseForwardHeaders() { + static boolean doUseForwardHeaders() { return Boolean.TRUE.equals(ApiServer.useForwardHeader.value()); } - String[] proxyNets() { + static String[] proxyNets() { return ApiServer.proxyForwardList.value().split(","); } //This method will try to get login IP of user even if servlet is behind reverseProxy or loadBalancer - public InetAddress getClientAddress(final HttpServletRequest request) throws UnknownHostException { - String ip = null; - InetAddress pretender = InetAddress.getByName(request.getRemoteAddr()); - if(doUseForwardHeaders()) { - if (NetUtils.isIpInCidrList(pretender, proxyNets())) { + public static InetAddress getClientAddress(final HttpServletRequest request) throws UnknownHostException { + final String remote = request.getRemoteAddr(); + if (doUseForwardHeaders()) { + final InetAddress remoteAddr = InetAddress.getByName(remote); + if (NetUtils.isIpInCidrList(remoteAddr, proxyNets())) { for (String header : getClientAddressHeaders()) { header = header.trim(); - ip = getCorrectIPAddress(request.getHeader(header)); + final String ip = getCorrectIPAddress(request.getHeader(header)); if (StringUtils.isNotBlank(ip)) { - LOGGER.debug(String.format("found ip %s in header %s ", ip, header)); - break; + LOGGER.debug("found ip {} in header {}", ip, header); + return InetAddress.getByName(ip); } - } // no address found in header so ip is blank and use remote addr - } // else not an allowed proxy address, ip is blank and use remote addr + } + } } - if (StringUtils.isBlank(ip)) { - LOGGER.trace(String.format("no ip found in headers, returning remote address %s.", pretender.getHostAddress())); - return pretender; - } - - return InetAddress.getByName(ip); + LOGGER.trace("no ip found in headers, returning remote address {}.", remote); + return InetAddress.getByName(remote); } - private String[] getClientAddressHeaders() { + private static String[] getClientAddressHeaders() { return ApiServer.listOfForwardHeaders.value().split(","); } diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java index 472be9a12b7..6713a7dc07e 100644 --- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java @@ -2425,6 +2425,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q Long msId = cmd.getManagementServerId(); final CPU.CPUArch arch = cmd.getArch(); String storageAccessGroup = cmd.getStorageAccessGroup(); + String version = cmd.getVersion(); Filter searchFilter = new Filter(HostVO.class, "id", Boolean.TRUE, startIndex, pageSize); @@ -2449,11 +2450,13 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q hostSearchBuilder.or("storageAccessGroupMiddle", hostSearchBuilder.entity().getStorageAccessGroups(), Op.LIKE); hostSearchBuilder.cp(); } + hostSearchBuilder.and("version", hostSearchBuilder.entity().getVersion(), SearchCriteria.Op.EQ); if (keyword != null) { hostSearchBuilder.and().op("keywordName", hostSearchBuilder.entity().getName(), SearchCriteria.Op.LIKE); hostSearchBuilder.or("keywordStatus", hostSearchBuilder.entity().getStatus(), SearchCriteria.Op.LIKE); hostSearchBuilder.or("keywordType", hostSearchBuilder.entity().getType(), SearchCriteria.Op.LIKE); + hostSearchBuilder.or("keywordVersion", hostSearchBuilder.entity().getVersion(), SearchCriteria.Op.LIKE); hostSearchBuilder.cp(); } @@ -2484,6 +2487,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q sc.setParameters("keywordName", "%" + keyword + "%"); sc.setParameters("keywordStatus", "%" + keyword + "%"); sc.setParameters("keywordType", "%" + keyword + "%"); + sc.setParameters("keywordVersion", "%" + keyword + "%"); } if (id != null) { @@ -2547,6 +2551,10 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q sc.setParameters("storageAccessGroupMiddle", "%," + storageAccessGroup + ",%"); } + if (version != null) { + sc.setParameters("version", version); + } + Pair, Integer> uniqueHostPair = hostDao.searchAndCount(sc, searchFilter); Integer count = uniqueHostPair.second(); List hostIds = uniqueHostPair.first().stream().map(HostVO::getId).collect(Collectors.toList()); @@ -2860,6 +2868,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q boolean listAll = cmd.listAll(); boolean isRecursive = false; Domain domain = null; + Map tags = cmd.getTags(); if (domainId != null) { domain = _domainDao.findById(domainId); @@ -2889,6 +2898,24 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q domainSearchBuilder.and("path", domainSearchBuilder.entity().getPath(), SearchCriteria.Op.LIKE); domainSearchBuilder.and("state", domainSearchBuilder.entity().getState(), SearchCriteria.Op.EQ); + if (MapUtils.isNotEmpty(tags)) { + SearchBuilder resourceTagSearch = resourceTagDao.createSearchBuilder(); + resourceTagSearch.and("resourceType", resourceTagSearch.entity().getResourceType(), Op.EQ); + resourceTagSearch.and().op(); + for (int count = 0; count < tags.size(); count++) { + if (count == 0) { + resourceTagSearch.op("tagKey" + count, resourceTagSearch.entity().getKey(), Op.EQ); + } else { + resourceTagSearch.or().op("tagKey" + count, resourceTagSearch.entity().getKey(), Op.EQ); + } + resourceTagSearch.and("tagValue" + count, resourceTagSearch.entity().getValue(), Op.EQ); + resourceTagSearch.cp(); + } + resourceTagSearch.cp(); + + domainSearchBuilder.join("tags", resourceTagSearch, resourceTagSearch.entity().getResourceId(), domainSearchBuilder.entity().getId(), JoinBuilder.JoinType.INNER); + } + if (keyword != null) { domainSearchBuilder.and("keywordName", domainSearchBuilder.entity().getName(), SearchCriteria.Op.LIKE); } @@ -2918,6 +2945,16 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q } } + if (MapUtils.isNotEmpty(tags)) { + int count = 0; + sc.setJoinParameters("tags", "resourceType", ResourceObjectType.Domain); + for (Map.Entry entry : tags.entrySet()) { + sc.setJoinParameters("tags", "tagKey" + count, entry.getKey()); + sc.setJoinParameters("tags", "tagValue" + count, entry.getValue()); + count++; + } + } + // return only Active domains to the API sc.setParameters("state", Domain.State.Active); @@ -5390,6 +5427,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q options.put(VmDetailConstants.VIRTUAL_TPM_VERSION, Arrays.asList("1.2", "2.0")); options.put(VmDetailConstants.GUEST_CPU_MODE, Arrays.asList("custom", "host-model", "host-passthrough")); options.put(VmDetailConstants.GUEST_CPU_MODEL, Collections.emptyList()); + options.put(VmDetailConstants.KVM_GUEST_OS_MACHINE_TYPE, Collections.emptyList()); options.put(VmDetailConstants.KVM_SKIP_FORCE_DISK_CONTROLLER, Arrays.asList("true", "false")); } @@ -5402,6 +5440,10 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q options.put(VmDetailConstants.RAM_RESERVATION, Collections.emptyList()); options.put(VmDetailConstants.VIRTUAL_TPM_ENABLED, Arrays.asList("true", "false")); } + + if (HypervisorType.XenServer.equals(hypervisorType)) { + options.put(VmDetailConstants.VIRTUAL_TPM_ENABLED, Arrays.asList("true", "false")); + } } @Override @@ -5715,6 +5757,8 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q protected Pair, Integer> listManagementServersInternal(ListMgmtsCmd cmd) { Long id = cmd.getId(); String name = cmd.getHostName(); + String version = cmd.getVersion(); + String keyword = cmd.getKeyword(); SearchBuilder sb = managementServerJoinDao.createSearchBuilder(); SearchCriteria sc = sb.create(); @@ -5724,6 +5768,12 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q if (name != null) { sc.addAnd("name", SearchCriteria.Op.EQ, name); } + if (version != null) { + sc.addAnd("version", SearchCriteria.Op.EQ, version); + } + if (keyword != null) { + sc.addAnd("version", SearchCriteria.Op.LIKE, "%" + keyword + "%"); + } return managementServerJoinDao.searchAndCount(sc, null); } diff --git a/server/src/main/java/com/cloud/api/query/dao/DomainJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/DomainJoinDaoImpl.java index d4865c5550e..b6a3370e6bc 100644 --- a/server/src/main/java/com/cloud/api/query/dao/DomainJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/DomainJoinDaoImpl.java @@ -20,10 +20,8 @@ import java.util.ArrayList; import java.util.EnumSet; import java.util.List; +import javax.inject.Inject; -import com.cloud.api.ApiResponseHelper; -import com.cloud.configuration.Resource; -import com.cloud.user.AccountManager; import org.apache.cloudstack.annotation.AnnotationService; import org.apache.cloudstack.annotation.dao.AnnotationDao; import org.apache.cloudstack.api.ApiConstants.DomainDetails; @@ -35,15 +33,16 @@ import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.springframework.stereotype.Component; import com.cloud.api.ApiDBUtils; +import com.cloud.api.ApiResponseHelper; import com.cloud.api.query.vo.DomainJoinVO; +import com.cloud.configuration.Resource; import com.cloud.configuration.Resource.ResourceType; import com.cloud.domain.Domain; +import com.cloud.user.AccountManager; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; -import javax.inject.Inject; - @Component public class DomainJoinDaoImpl extends GenericDaoBase implements DomainJoinDao { @@ -110,6 +109,7 @@ public class DomainJoinDaoImpl extends GenericDaoBase implem } domainResponse.setDetails(ApiDBUtils.getDomainDetails(domain.getId())); + ApiResponseHelper.populateDomainTags(domain.getUuid(), domainResponse); domainResponse.setObjectName("domain"); return domainResponse; diff --git a/server/src/main/java/com/cloud/api/query/dao/SnapshotJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/SnapshotJoinDaoImpl.java index 9ea14edf2b7..0b0d9b269f3 100644 --- a/server/src/main/java/com/cloud/api/query/dao/SnapshotJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/SnapshotJoinDaoImpl.java @@ -109,6 +109,8 @@ public class SnapshotJoinDaoImpl extends GenericDaoBaseWithTagInformation isosInStore = _templateStoreDao.listByTemplateNotBypassed(iso.getId()); diff --git a/server/src/main/java/com/cloud/capacity/CapacityManagerImpl.java b/server/src/main/java/com/cloud/capacity/CapacityManagerImpl.java index 2de9abc827e..2940f900b08 100644 --- a/server/src/main/java/com/cloud/capacity/CapacityManagerImpl.java +++ b/server/src/main/java/com/cloud/capacity/CapacityManagerImpl.java @@ -50,7 +50,6 @@ import com.cloud.agent.api.AgentControlCommand; import com.cloud.agent.api.Answer; import com.cloud.agent.api.Command; import com.cloud.agent.api.StartupCommand; -import com.cloud.agent.api.StartupRoutingCommand; import com.cloud.capacity.dao.CapacityDao; import com.cloud.configuration.Config; import com.cloud.dc.ClusterDetailsDao; @@ -82,7 +81,6 @@ import com.cloud.utils.NumbersUtil; import com.cloud.utils.Pair; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.db.DB; -import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.Transaction; import com.cloud.utils.db.TransactionCallbackNoReturn; import com.cloud.utils.db.TransactionStatus; @@ -167,11 +165,6 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager, return true; } - @Override - public boolean stop() { - return true; - } - @DB @Override public boolean releaseVmCapacity(VirtualMachine vm, final boolean moveFromReserved, final boolean moveToReservered, final Long hostId) { @@ -395,8 +388,8 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager, long cluster_id = host.getClusterId(); ClusterDetailsVO cluster_detail_cpu = _clusterDetailsDao.findDetail(cluster_id, VmDetailConstants.CPU_OVER_COMMIT_RATIO); ClusterDetailsVO cluster_detail_ram = _clusterDetailsDao.findDetail(cluster_id, VmDetailConstants.MEMORY_OVER_COMMIT_RATIO); - Float cpuOvercommitRatio = Float.parseFloat(cluster_detail_cpu.getValue()); - Float memoryOvercommitRatio = Float.parseFloat(cluster_detail_ram.getValue()); + float cpuOvercommitRatio = Float.parseFloat(cluster_detail_cpu.getValue()); + float memoryOvercommitRatio = Float.parseFloat(cluster_detail_ram.getValue()); boolean hostHasCpuCapability, hostHasCapacity = false; hostHasCpuCapability = checkIfHostHasCpuCapability(host, cpucore, cpuspeed); @@ -424,14 +417,13 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager, if (e instanceof CloudRuntimeException) { throw e; } - return; } } @Override public boolean checkIfHostHasCpuCapability(Host host, Integer cpuNum, Integer cpuSpeed) { // Check host can support the Cpu Number and Speed. - boolean isCpuNumGood = host.getCpus().intValue() >= cpuNum; + boolean isCpuNumGood = host.getCpus() >= cpuNum; boolean isCpuSpeedGood = host.getSpeed().intValue() >= cpuSpeed; boolean hasCpuCapability = isCpuNumGood && isCpuSpeedGood; @@ -482,13 +474,10 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager, String failureReason = ""; if (checkFromReservedCapacity) { - long freeCpu = reservedCpu; - long freeMem = reservedMem; - if (logger.isDebugEnabled()) { logger.debug("We need to allocate to the last host again, so checking if there is enough reserved capacity"); - logger.debug("Reserved CPU: " + freeCpu + " , Requested CPU: " + cpu); - logger.debug("Reserved RAM: " + toHumanReadableSize(freeMem) + " , Requested RAM: " + toHumanReadableSize(ram)); + logger.debug("Reserved CPU: " + reservedCpu + " , Requested CPU: " + cpu); + logger.debug("Reserved RAM: " + toHumanReadableSize(reservedMem) + " , Requested RAM: " + toHumanReadableSize(ram)); } /* alloc from reserved */ if (reservedCpu >= cpu) { @@ -586,7 +575,7 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager, @Override public long getAllocatedPoolCapacity(StoragePoolVO pool, VMTemplateVO templateForVmCreation) { - long totalAllocatedSize = 0; + long totalAllocatedSize; // if the storage pool is managed, the used bytes can be larger than the sum of the sizes of all of the non-destroyed volumes // in this case, call getUsedBytes(StoragePoolVO) @@ -700,11 +689,11 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager, Pair clusterValues = clusterValuesCache.get(host.getClusterId()); - Float clusterCpuOvercommitRatio = Float.parseFloat(clusterValues.first()); - Float clusterRamOvercommitRatio = Float.parseFloat(clusterValues.second()); + float clusterCpuOvercommitRatio = Float.parseFloat(clusterValues.first()); + float clusterRamOvercommitRatio = Float.parseFloat(clusterValues.second()); for (VMInstanceVO vm : vms) { - Float cpuOvercommitRatio = 1.0f; - Float ramOvercommitRatio = 1.0f; + float cpuOvercommitRatio; + float ramOvercommitRatio; Map vmDetails = getVmDetailsForCapacityCalculation(vm.getId()); String vmDetailCpu = vmDetails.get(VmDetailConstants.CPU_OVER_COMMIT_RATIO); String vmDetailRam = vmDetails.get(VmDetailConstants.MEMORY_OVER_COMMIT_RATIO); @@ -717,21 +706,22 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager, } if (so.isDynamic()) { usedMemory += - ((Integer.parseInt(vmDetails.get(UsageEventVO.DynamicParameters.memory.name())) * 1024L * 1024L) / ramOvercommitRatio) * - clusterRamOvercommitRatio; + (long) (((Integer.parseInt(vmDetails.get(UsageEventVO.DynamicParameters.memory.name())) * 1024L * 1024L) + / ramOvercommitRatio) * clusterRamOvercommitRatio); if(vmDetails.containsKey(UsageEventVO.DynamicParameters.cpuSpeed.name())) { usedCpu += - ((Integer.parseInt(vmDetails.get(UsageEventVO.DynamicParameters.cpuNumber.name())) * Integer.parseInt(vmDetails.get(UsageEventVO.DynamicParameters.cpuSpeed.name()))) / cpuOvercommitRatio) * - clusterCpuOvercommitRatio; + (long) ((((long) Integer.parseInt(vmDetails.get(UsageEventVO.DynamicParameters.cpuNumber.name())) + * Integer.parseInt(vmDetails.get(UsageEventVO.DynamicParameters.cpuSpeed.name()))) + / cpuOvercommitRatio) * clusterCpuOvercommitRatio); } else { usedCpu += - ((Integer.parseInt(vmDetails.get(UsageEventVO.DynamicParameters.cpuNumber.name())) * so.getSpeed()) / cpuOvercommitRatio) * - clusterCpuOvercommitRatio; + (long) ((((long) Integer.parseInt(vmDetails.get(UsageEventVO.DynamicParameters.cpuNumber.name())) * so.getSpeed()) / cpuOvercommitRatio) * + clusterCpuOvercommitRatio); } usedCpuCore += Integer.parseInt(vmDetails.get(UsageEventVO.DynamicParameters.cpuNumber.name())); } else { - usedMemory += ((so.getRamSize() * 1024L * 1024L) / ramOvercommitRatio) * clusterRamOvercommitRatio; - usedCpu += ((so.getCpu() * so.getSpeed()) / cpuOvercommitRatio) * clusterCpuOvercommitRatio; + usedMemory += (long) (((so.getRamSize() * 1024L * 1024L) / ramOvercommitRatio) * clusterRamOvercommitRatio); + usedCpu += (long) ((((long) so.getCpu() * so.getSpeed()) / cpuOvercommitRatio) * clusterCpuOvercommitRatio); usedCpuCore += so.getCpu(); } } @@ -740,8 +730,8 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager, logger.debug("Found {} VM, not running on {}", vmsByLastHostId.size(), host); for (VMInstanceVO vm : vmsByLastHostId) { - Float cpuOvercommitRatio = 1.0f; - Float ramOvercommitRatio = 1.0f; + float cpuOvercommitRatio = 1.0f; + float ramOvercommitRatio = 1.0f; long lastModificationTime = Optional.ofNullable(vm.getUpdateTime()).orElse(vm.getCreated()).getTime(); long secondsSinceLastUpdate = (DateUtil.currentGMTTime().getTime() - lastModificationTime) / 1000; if (secondsSinceLastUpdate < _vmCapacityReleaseInterval) { @@ -761,28 +751,28 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager, } if (so.isDynamic()) { reservedMemory += - ((Integer.parseInt(vmDetails.get(UsageEventVO.DynamicParameters.memory.name())) * 1024L * 1024L) / ramOvercommitRatio) * - clusterRamOvercommitRatio; + (long) (((Integer.parseInt(vmDetails.get(UsageEventVO.DynamicParameters.memory.name())) * 1024L * 1024L) / ramOvercommitRatio) * + clusterRamOvercommitRatio); if(vmDetails.containsKey(UsageEventVO.DynamicParameters.cpuSpeed.name())) { reservedCpu += - ((Integer.parseInt(vmDetails.get(UsageEventVO.DynamicParameters.cpuNumber.name())) * Integer.parseInt(vmDetails.get(UsageEventVO.DynamicParameters.cpuSpeed.name()))) / cpuOvercommitRatio) * - clusterCpuOvercommitRatio; + (long) (((Long.parseLong(vmDetails.get(UsageEventVO.DynamicParameters.cpuNumber.name())) * Integer.parseInt(vmDetails.get(UsageEventVO.DynamicParameters.cpuSpeed.name()))) / cpuOvercommitRatio) * + clusterCpuOvercommitRatio); } else { reservedCpu += - ((Integer.parseInt(vmDetails.get(UsageEventVO.DynamicParameters.cpuNumber.name())) * so.getSpeed()) / cpuOvercommitRatio) * - clusterCpuOvercommitRatio; + (long) (((Long.parseLong(vmDetails.get(UsageEventVO.DynamicParameters.cpuNumber.name())) * so.getSpeed()) / cpuOvercommitRatio) * + clusterCpuOvercommitRatio); } reservedCpuCore += Integer.parseInt(vmDetails.get(UsageEventVO.DynamicParameters.cpuNumber.name())); } else { - reservedMemory += ((so.getRamSize() * 1024L * 1024L) / ramOvercommitRatio) * clusterRamOvercommitRatio; - reservedCpu += (so.getCpu() * so.getSpeed() / cpuOvercommitRatio) * clusterCpuOvercommitRatio; + reservedMemory += (long) (((so.getRamSize() * 1024L * 1024L) / ramOvercommitRatio) * clusterRamOvercommitRatio); + reservedCpu += (long) (((long) so.getCpu() * so.getSpeed() / cpuOvercommitRatio) * clusterCpuOvercommitRatio); reservedCpuCore += so.getCpu(); } } else { // signal if not done already, that the VM has been stopped for skip.counting.hours, // hence capacity will not be reserved anymore. VMInstanceDetailVO messageSentFlag = _vmInstanceDetailsDao.findDetail(vm.getId(), VmDetailConstants.MESSAGE_RESERVED_CAPACITY_FREED_FLAG); - if (messageSentFlag == null || !Boolean.valueOf(messageSentFlag.getValue())) { + if (messageSentFlag == null || !Boolean.parseBoolean(messageSentFlag.getValue())) { _messageBus.publish(_name, "VM_ReservedCapacity_Free", PublishScope.LOCAL, vm); if (vm.getType() == VirtualMachine.Type.User) { @@ -859,7 +849,7 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager, if (host.getTotalMemory() != null) { memCap.setTotalCapacity(host.getTotalMemory()); } - long hostTotalCpu = host.getCpus().longValue() * host.getSpeed().longValue(); + long hostTotalCpu = host.getCpus().longValue() * host.getSpeed(); if (cpuCap.getTotalCapacity() != hostTotalCpu) { logger.debug("Calibrate total cpu for host: {} old total CPU:{} new total CPU:{}", host, cpuCap.getTotalCapacity(), hostTotalCpu); @@ -938,7 +928,7 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager, capacity = new CapacityVO(host.getId(), host.getDataCenterId(), host.getPodId(), host.getClusterId(), usedCpuFinal, host.getCpus().longValue() * - host.getSpeed().longValue(), Capacity.CAPACITY_TYPE_CPU); + host.getSpeed(), Capacity.CAPACITY_TYPE_CPU); capacity.setReservedCapacity(reservedCpuFinal); capacity.setCapacityState(capacityState); _capacityDao.persist(capacity); @@ -1029,78 +1019,10 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager, return true; } - // TODO: Get rid of this case once we've determined that the capacity listeners above have all the changes - // create capacity entries if none exist for this server - private void createCapacityEntry(StartupCommand startup, HostVO server) { - SearchCriteria capacitySC = _capacityDao.createSearchCriteria(); - capacitySC.addAnd("hostOrPoolId", SearchCriteria.Op.EQ, server.getId()); - capacitySC.addAnd("dataCenterId", SearchCriteria.Op.EQ, server.getDataCenterId()); - capacitySC.addAnd("podId", SearchCriteria.Op.EQ, server.getPodId()); - - if (startup instanceof StartupRoutingCommand) { - SearchCriteria capacityCPU = _capacityDao.createSearchCriteria(); - capacityCPU.addAnd("hostOrPoolId", SearchCriteria.Op.EQ, server.getId()); - capacityCPU.addAnd("dataCenterId", SearchCriteria.Op.EQ, server.getDataCenterId()); - capacityCPU.addAnd("podId", SearchCriteria.Op.EQ, server.getPodId()); - capacityCPU.addAnd("capacityType", SearchCriteria.Op.EQ, Capacity.CAPACITY_TYPE_CPU); - List capacityVOCpus = _capacityDao.search(capacitySC, null); - Float cpuovercommitratio = Float.parseFloat(_clusterDetailsDao.findDetail(server.getClusterId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO).getValue()); - Float memoryOvercommitRatio = Float.parseFloat(_clusterDetailsDao.findDetail(server.getClusterId(), VmDetailConstants.MEMORY_OVER_COMMIT_RATIO).getValue()); - - if (capacityVOCpus != null && !capacityVOCpus.isEmpty()) { - CapacityVO CapacityVOCpu = capacityVOCpus.get(0); - long newTotalCpu = (long)(server.getCpus().longValue() * server.getSpeed().longValue() * cpuovercommitratio); - if ((CapacityVOCpu.getTotalCapacity() <= newTotalCpu) || ((CapacityVOCpu.getUsedCapacity() + CapacityVOCpu.getReservedCapacity()) <= newTotalCpu)) { - CapacityVOCpu.setTotalCapacity(newTotalCpu); - } else if ((CapacityVOCpu.getUsedCapacity() + CapacityVOCpu.getReservedCapacity() > newTotalCpu) && (CapacityVOCpu.getUsedCapacity() < newTotalCpu)) { - CapacityVOCpu.setReservedCapacity(0); - CapacityVOCpu.setTotalCapacity(newTotalCpu); - } else { - logger.debug("What? new cpu is :" + newTotalCpu + ", old one is " + CapacityVOCpu.getUsedCapacity() + "," + CapacityVOCpu.getReservedCapacity() + - "," + CapacityVOCpu.getTotalCapacity()); - } - _capacityDao.update(CapacityVOCpu.getId(), CapacityVOCpu); - } else { - CapacityVO capacity = - new CapacityVO(server.getId(), server.getDataCenterId(), server.getPodId(), server.getClusterId(), 0L, server.getCpus().longValue() * - server.getSpeed().longValue(), Capacity.CAPACITY_TYPE_CPU); - _capacityDao.persist(capacity); - } - - SearchCriteria capacityMem = _capacityDao.createSearchCriteria(); - capacityMem.addAnd("hostOrPoolId", SearchCriteria.Op.EQ, server.getId()); - capacityMem.addAnd("dataCenterId", SearchCriteria.Op.EQ, server.getDataCenterId()); - capacityMem.addAnd("podId", SearchCriteria.Op.EQ, server.getPodId()); - capacityMem.addAnd("capacityType", SearchCriteria.Op.EQ, Capacity.CAPACITY_TYPE_MEMORY); - List capacityVOMems = _capacityDao.search(capacityMem, null); - - if (capacityVOMems != null && !capacityVOMems.isEmpty()) { - CapacityVO CapacityVOMem = capacityVOMems.get(0); - long newTotalMem = (long)((server.getTotalMemory()) * memoryOvercommitRatio); - if (CapacityVOMem.getTotalCapacity() <= newTotalMem || (CapacityVOMem.getUsedCapacity() + CapacityVOMem.getReservedCapacity() <= newTotalMem)) { - CapacityVOMem.setTotalCapacity(newTotalMem); - } else if (CapacityVOMem.getUsedCapacity() + CapacityVOMem.getReservedCapacity() > newTotalMem && CapacityVOMem.getUsedCapacity() < newTotalMem) { - CapacityVOMem.setReservedCapacity(0); - CapacityVOMem.setTotalCapacity(newTotalMem); - } else { - logger.debug("What? new mem is :" + newTotalMem + ", old one is " + CapacityVOMem.getUsedCapacity() + "," + CapacityVOMem.getReservedCapacity() + - "," + CapacityVOMem.getTotalCapacity()); - } - _capacityDao.update(CapacityVOMem.getId(), CapacityVOMem); - } else { - CapacityVO capacity = - new CapacityVO(server.getId(), server.getDataCenterId(), server.getPodId(), server.getClusterId(), 0L, server.getTotalMemory(), - Capacity.CAPACITY_TYPE_MEMORY); - _capacityDao.persist(capacity); - } - } - - } - @Override public float getClusterOverProvisioningFactor(Long clusterId, short capacityType) { - String capacityOverProvisioningName = ""; + String capacityOverProvisioningName; if (capacityType == Capacity.CAPACITY_TYPE_CPU) { capacityOverProvisioningName = VmDetailConstants.CPU_OVER_COMMIT_RATIO; } else if (capacityType == Capacity.CAPACITY_TYPE_MEMORY) { @@ -1110,15 +1032,14 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager, } ClusterDetailsVO clusterDetailCpu = _clusterDetailsDao.findDetail(clusterId, capacityOverProvisioningName); - Float clusterOverProvisioningRatio = Float.parseFloat(clusterDetailCpu.getValue()); - return clusterOverProvisioningRatio; + return Float.parseFloat(clusterDetailCpu.getValue()); } @Override public boolean checkIfClusterCrossesThreshold(Long clusterId, Integer cpuRequested, long ramRequested) { - Float clusterCpuOverProvisioning = getClusterOverProvisioningFactor(clusterId, Capacity.CAPACITY_TYPE_CPU); - Float clusterMemoryOverProvisioning = getClusterOverProvisioningFactor(clusterId, Capacity.CAPACITY_TYPE_MEMORY); + float clusterCpuOverProvisioning = getClusterOverProvisioningFactor(clusterId, Capacity.CAPACITY_TYPE_CPU); + float clusterMemoryOverProvisioning = getClusterOverProvisioningFactor(clusterId, Capacity.CAPACITY_TYPE_MEMORY); Float clusterCpuCapacityDisableThreshold = DeploymentClusterPlanner.ClusterCPUCapacityDisableThreshold.valueIn(clusterId); Float clusterMemoryCapacityDisableThreshold = DeploymentClusterPlanner.ClusterMemoryCapacityDisableThreshold.valueIn(clusterId); @@ -1148,8 +1069,8 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager, int cpu_requested = offering.getCpu() * offering.getSpeed(); long ram_requested = offering.getRamSize() * 1024L * 1024L; Pair clusterDetails = getClusterValues(host.getClusterId()); - Float cpuOvercommitRatio = Float.parseFloat(clusterDetails.first()); - Float memoryOvercommitRatio = Float.parseFloat(clusterDetails.second()); + float cpuOvercommitRatio = Float.parseFloat(clusterDetails.first()); + float memoryOvercommitRatio = Float.parseFloat(clusterDetails.second()); boolean hostHasCpuCapability = checkIfHostHasCpuCapability(host, offering.getCpu(), offering.getSpeed()); boolean hostHasCapacity = checkIfHostHasCapacity(host, cpu_requested, ram_requested, false, cpuOvercommitRatio, memoryOvercommitRatio, diff --git a/server/src/main/java/com/cloud/configuration/Config.java b/server/src/main/java/com/cloud/configuration/Config.java index abae4d3996c..a82ccb33a4a 100644 --- a/server/src/main/java/com/cloud/configuration/Config.java +++ b/server/src/main/java/com/cloud/configuration/Config.java @@ -542,13 +542,6 @@ public enum Config { "true", "Indicates whether or not to automatically reserver system VM standby capacity.", null), - SystemVMDefaultHypervisor("Advanced", - ManagementServer.class, - String.class, - "system.vm.default.hypervisor", - null, - "Hypervisor type used to create system vm, valid values are: XenServer, KVM, VMware, Hyperv, VirtualBox, Parralels, BareMetal, Ovm, LXC, Any", - null), SystemVMRandomPassword( "Advanced", ManagementServer.class, diff --git a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java index 1798101bb4c..b47b12aa1a5 100644 --- a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java +++ b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java @@ -545,7 +545,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati public static final ConfigKey DELETE_QUERY_BATCH_SIZE = new ConfigKey<>("Advanced", Long.class, "delete.query.batch.size", "0", "Indicates the limit applied while deleting entries in bulk. With this, the delete query will apply the limit as many times as necessary," + " to delete all the entries. This is advised when retaining several days of records, which can lead to slowness. <= 0 means that no limit will " + - "be applied. Default value is 0. For now, this is used for deletion of vm & volume stats only.", true); + "be applied. Default value is 0. For now, this is used for deletion of VM stats, volume stats, and usage records.", true); private static final String IOPS_READ_RATE = "IOPS Read"; private static final String IOPS_WRITE_RATE = "IOPS Write"; diff --git a/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java b/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java index bb165e0529d..230f97f6fd3 100644 --- a/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java +++ b/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java @@ -37,6 +37,7 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; import com.cloud.gpu.dao.VgpuProfileDao; +import com.cloud.resource.ResourceState; import org.apache.cloudstack.affinity.AffinityGroupDomainMapVO; import org.apache.cloudstack.affinity.AffinityGroupProcessor; import org.apache.cloudstack.affinity.AffinityGroupService; @@ -383,22 +384,12 @@ StateListener, Configurable { planner = getDeploymentPlannerByName(plannerName); } - Host lastHost = null; - - String considerLastHostStr = (String)vmProfile.getParameter(VirtualMachineProfile.Param.ConsiderLastHost); - boolean considerLastHost = vm.getLastHostId() != null && haVmTag == null && - (considerLastHostStr == null || Boolean.TRUE.toString().equalsIgnoreCase(considerLastHostStr)); - if (considerLastHost) { - HostVO host = _hostDao.findById(vm.getLastHostId()); - logger.debug("This VM has last host_id specified, trying to choose the same host: " + host); - lastHost = host; - - DeployDestination deployDestination = deployInVmLastHost(vmProfile, plan, avoids, planner, vm, dc, offering, cpuRequested, ramRequested, volumesRequireEncryption); - if (deployDestination != null) { - return deployDestination; - } + DeployDestination deployDestinationForVmLasthost = deployInVmLastHost(vmProfile, plan, avoids, planner, vm, dc, offering, cpuRequested, ramRequested, volumesRequireEncryption); + if (deployDestinationForVmLasthost != null) { + return deployDestinationForVmLasthost; } + HostVO lastHost = _hostDao.findById(vm.getLastHostId()); avoidOtherClustersForDeploymentIfMigrationDisabled(vm, lastHost, avoids); DeployDestination dest = null; @@ -480,47 +471,56 @@ StateListener, Configurable { private DeployDestination deployInVmLastHost(VirtualMachineProfile vmProfile, DeploymentPlan plan, ExcludeList avoids, DeploymentPlanner planner, VirtualMachine vm, DataCenter dc, ServiceOffering offering, int cpuRequested, long ramRequested, boolean volumesRequireEncryption) throws InsufficientServerCapacityException { - HostVO host = _hostDao.findById(vm.getLastHostId()); - if (canUseLastHost(host, avoids, plan, vm, offering, volumesRequireEncryption)) { - _hostDao.loadHostTags(host); - _hostDao.loadDetails(host); - if (host.getStatus() != Status.Up) { + String considerLastHostStr = (String)vmProfile.getParameter(VirtualMachineProfile.Param.ConsiderLastHost); + String haVmTag = (String)vmProfile.getParameter(VirtualMachineProfile.Param.HaTag); + boolean considerLastHost = vm.getLastHostId() != null && haVmTag == null && + !(Boolean.FALSE.toString().equalsIgnoreCase(considerLastHostStr)); + if (!considerLastHost) { + return null; + } + + logger.debug("This VM has last host_id: {}", vm.getLastHostId()); + HostVO lastHost = _hostDao.findById(vm.getLastHostId()); + if (canUseLastHost(lastHost, avoids, plan, vm, offering, volumesRequireEncryption)) { + _hostDao.loadHostTags(lastHost); + _hostDao.loadDetails(lastHost); + if (lastHost.getStatus() != Status.Up) { logger.debug("Cannot deploy VM [{}] to the last host [{}] because this host is not in UP state or is not enabled. Host current status [{}] and resource status [{}].", - vm, host, host.getState().name(), host.getResourceState()); + vm, lastHost, lastHost.getState().name(), lastHost.getResourceState()); return null; } - if (checkVmProfileAndHost(vmProfile, host)) { - long cluster_id = host.getClusterId(); + if (checkVmProfileAndHost(vmProfile, lastHost)) { + long cluster_id = lastHost.getClusterId(); ClusterDetailsVO cluster_detail_cpu = _clusterDetailsDao.findDetail(cluster_id, "cpuOvercommitRatio"); ClusterDetailsVO cluster_detail_ram = _clusterDetailsDao.findDetail(cluster_id, "memoryOvercommitRatio"); float cpuOvercommitRatio = Float.parseFloat(cluster_detail_cpu.getValue()); float memoryOvercommitRatio = Float.parseFloat(cluster_detail_ram.getValue()); boolean hostHasCpuCapability, hostHasCapacity = false; - hostHasCpuCapability = _capacityMgr.checkIfHostHasCpuCapability(host, offering.getCpu(), offering.getSpeed()); + hostHasCpuCapability = _capacityMgr.checkIfHostHasCpuCapability(lastHost, offering.getCpu(), offering.getSpeed()); if (hostHasCpuCapability) { // first check from reserved capacity - hostHasCapacity = _capacityMgr.checkIfHostHasCapacity(host, cpuRequested, ramRequested, true, cpuOvercommitRatio, memoryOvercommitRatio, true); + hostHasCapacity = _capacityMgr.checkIfHostHasCapacity(lastHost, cpuRequested, ramRequested, true, cpuOvercommitRatio, memoryOvercommitRatio, true); // if not reserved, check the free capacity if (!hostHasCapacity) - hostHasCapacity = _capacityMgr.checkIfHostHasCapacity(host, cpuRequested, ramRequested, false, cpuOvercommitRatio, memoryOvercommitRatio, true); + hostHasCapacity = _capacityMgr.checkIfHostHasCapacity(lastHost, cpuRequested, ramRequested, false, cpuOvercommitRatio, memoryOvercommitRatio, true); } boolean displayStorage = getDisplayStorageFromVmProfile(vmProfile); if (!hostHasCapacity || !hostHasCpuCapability) { - logger.debug("Cannot deploy VM [{}] to the last host [{}] because this host does not have enough capacity to deploy this VM.", vm, host); + logger.debug("Cannot deploy VM [{}] to the last host [{}] because this host does not have enough capacity to deploy this VM.", vm, lastHost); return null; } - Pod pod = _podDao.findById(host.getPodId()); - Cluster cluster = _clusterDao.findById(host.getClusterId()); + Pod pod = _podDao.findById(lastHost.getPodId()); + Cluster cluster = _clusterDao.findById(lastHost.getClusterId()); logger.debug("Last host [{}] of VM [{}] is UP and has enough capacity. Checking for suitable pools for this host under zone [{}], pod [{}] and cluster [{}].", - host, vm, dc, pod, cluster); + lastHost, vm, dc, pod, cluster); - if (DEPLOYMENT_PLANNING_SKIP_HYPERVISORS.contains(vm.getHypervisorType())) { - DeployDestination dest = new DeployDestination(dc, pod, cluster, host, new HashMap<>(), displayStorage); + if (vm.getHypervisorType() == HypervisorType.BareMetal) { + DeployDestination dest = new DeployDestination(dc, pod, cluster, lastHost, new HashMap<>(), displayStorage); logger.debug("Returning Deployment Destination: {}.", dest); return dest; } @@ -528,8 +528,8 @@ StateListener, Configurable { // search for storage under the zone, pod, cluster // of // the last host. - DataCenterDeployment lastPlan = new DataCenterDeployment(host.getDataCenterId(), - host.getPodId(), host.getClusterId(), host.getId(), plan.getPoolId(), null); + DataCenterDeployment lastPlan = new DataCenterDeployment(lastHost.getDataCenterId(), + lastHost.getPodId(), lastHost.getClusterId(), lastHost.getId(), plan.getPoolId(), null); Pair>, List> result = findSuitablePoolsForVolumes( vmProfile, lastPlan, avoids, HostAllocator.RETURN_UPTO_ALL); Map> suitableVolumeStoragePools = result.first(); @@ -538,11 +538,11 @@ StateListener, Configurable { // choose the potential pool for this VM for this // host if (suitableVolumeStoragePools.isEmpty()) { - logger.debug("Cannot find suitable storage pools in host [{}] to deploy VM [{}]", host, vm); + logger.debug("Cannot find suitable storage pools in host [{}] to deploy VM [{}]", lastHost, vm); return null; } List suitableHosts = new ArrayList<>(); - suitableHosts.add(host); + suitableHosts.add(lastHost); Pair> potentialResources = findPotentialDeploymentResources( suitableHosts, suitableVolumeStoragePools, avoids, getPlannerUsage(planner, vmProfile, plan, avoids), readyAndReusedVolumes, plan.getPreferredHosts(), vm); @@ -555,7 +555,7 @@ StateListener, Configurable { for (Volume vol : readyAndReusedVolumes) { storageVolMap.remove(vol); } - DeployDestination dest = new DeployDestination(dc, pod, cluster, host, storageVolMap, displayStorage); + DeployDestination dest = new DeployDestination(dc, pod, cluster, lastHost, storageVolMap, displayStorage); logger.debug("Returning Deployment Destination: {}", dest); return dest; } @@ -567,7 +567,7 @@ StateListener, Configurable { private boolean canUseLastHost(HostVO host, ExcludeList avoids, DeploymentPlan plan, VirtualMachine vm, ServiceOffering offering, boolean volumesRequireEncryption) { if (host == null) { - logger.warn("Could not find last host of VM [{}] with id [{}]. Skipping this and trying other available hosts.", vm, vm.getLastHostId()); + logger.warn("Could not find last host of VM [{}] with id [{}]. Skipping it", vm, vm.getLastHostId()); return false; } @@ -581,6 +581,12 @@ StateListener, Configurable { return false; } + logger.debug("VM's last host is {}, trying to choose the same host if it is not in maintenance, error or degraded state", host); + if (host.isInMaintenanceStates() || Arrays.asList(ResourceState.Error, ResourceState.Degraded).contains(host.getResourceState())) { + logger.debug("Unable to deploy VM {} in the last host, last host {} is in {} state", vm.getName(), host.getName(), host.getResourceState()); + return false; + } + if (_capacityMgr.checkIfHostReachMaxGuestLimit(host)) { logger.debug("Cannot deploy VM [{}] in the last host [{}] because this host already has the max number of running VMs (users and system VMs). Skipping this and trying other available hosts.", vm, host); @@ -1477,7 +1483,7 @@ StateListener, Configurable { protected Pair> findPotentialDeploymentResources(List suitableHosts, Map> suitableVolumeStoragePools, ExcludeList avoid, PlannerResourceUsage resourceUsageRequired, List readyAndReusedVolumes, List preferredHosts, VirtualMachine vm) { - logger.debug("Trying to find a potenial host and associated storage pools from the suitable host/pool lists for this VM"); + logger.debug("Trying to find a potential host and associated storage pools from the suitable host/pool lists for this VM"); boolean hostCanAccessPool = false; boolean haveEnoughSpace = false; diff --git a/server/src/main/java/com/cloud/event/AlertGenerator.java b/server/src/main/java/com/cloud/event/AlertGenerator.java index f1b23e87308..601bf5e831a 100644 --- a/server/src/main/java/com/cloud/event/AlertGenerator.java +++ b/server/src/main/java/com/cloud/event/AlertGenerator.java @@ -67,12 +67,13 @@ public class AlertGenerator { } public static void publishAlertOnEventBus(String alertType, long dataCenterId, Long podId, String subject, String body) { - String configKey = Config.PublishAlertEvent.key(); String value = s_configDao.getValue(configKey); boolean configValue = Boolean.parseBoolean(value); - if(!configValue) + if (!configValue) { return; + } + try { eventDistributor = ComponentContext.getComponent(EventDistributor.class); } catch (NoSuchBeanDefinitionException nbe) { diff --git a/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java b/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java index e0292c890a2..c8635397b66 100644 --- a/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java +++ b/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java @@ -837,7 +837,7 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements Configur if (checkAndCancelWorkIfNeeded(work)) { return null; } - logger.info("Migration attempt: for VM {}from host {}. Starting attempt: {}/{} times.", vm, srcHost, 1 + work.getTimesTried(), _maxRetries); + logger.info("Migration attempt: for {} from {}. Starting attempt: {}/{} times.", vm, srcHost, 1 + work.getTimesTried(), _maxRetries); if (VirtualMachine.State.Stopped.equals(vm.getState())) { logger.info(String.format("vm %s is Stopped, skipping migrate.", vm)); @@ -847,8 +847,6 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements Configur logger.info(String.format("VM %s is running on a different host %s, skipping migration", vm, vm.getHostId())); return null; } - logger.info("Migration attempt: for VM " + vm.getUuid() + "from host id " + srcHostId + - ". Starting attempt: " + (1 + work.getTimesTried()) + "/" + _maxRetries + " times."); try { work.setStep(Step.Migrating); diff --git a/server/src/main/java/com/cloud/network/NetworkModelImpl.java b/server/src/main/java/com/cloud/network/NetworkModelImpl.java index 22d987a951f..07305c0e641 100644 --- a/server/src/main/java/com/cloud/network/NetworkModelImpl.java +++ b/server/src/main/java/com/cloud/network/NetworkModelImpl.java @@ -1443,11 +1443,11 @@ public class NetworkModelImpl extends ManagerBase implements NetworkModel, Confi return null; } + NetworkOffering offering = _entityMgr.findById(NetworkOffering.class, network.getNetworkOfferingId()); Long physicalNetworkId = null; if (effectiveTrafficType != TrafficType.Guest) { physicalNetworkId = getNonGuestNetworkPhysicalNetworkId(network, effectiveTrafficType); } else { - NetworkOffering offering = _entityMgr.findById(NetworkOffering.class, network.getNetworkOfferingId()); physicalNetworkId = network.getPhysicalNetworkId(); if (physicalNetworkId == null) { physicalNetworkId = findPhysicalNetworkId(network.getDataCenterId(), offering.getTags(), offering.getTrafficType()); @@ -1460,6 +1460,10 @@ public class NetworkModelImpl extends ManagerBase implements NetworkModel, Confi return null; } + if (offering != null && TrafficType.Guest.equals(offering.getTrafficType()) && offering.isSystemOnly()) { + // For private gateway, do not check the Guest traffic type + return _pNTrafficTypeDao.getNetworkTag(physicalNetworkId, null, hType); + } return _pNTrafficTypeDao.getNetworkTag(physicalNetworkId, effectiveTrafficType, hType); } diff --git a/server/src/main/java/com/cloud/network/NetworkServiceImpl.java b/server/src/main/java/com/cloud/network/NetworkServiceImpl.java index cb1ee021d67..dea8a7ec83e 100644 --- a/server/src/main/java/com/cloud/network/NetworkServiceImpl.java +++ b/server/src/main/java/com/cloud/network/NetworkServiceImpl.java @@ -1886,6 +1886,18 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C null, null, null, null, null, null, null, null, null, null); } + @Override + @DB + @ActionEvent(eventType = EventTypes.EVENT_NETWORK_CREATE, eventDescription = "creating network") + public Network createGuestNetwork(long networkOfferingId, String name, String displayText, Account owner, + PhysicalNetwork physicalNetwork, long zoneId, ACLType aclType, Pair vrIfaceMTUs) throws + InsufficientCapacityException, ConcurrentOperationException, ResourceAllocationException { + return _networkMgr.createGuestNetwork(networkOfferingId, name, displayText, + null, null, null, false, null, owner, null, physicalNetwork, zoneId, + aclType, null, null, null, null, true, null, + null, null, null, null, null, null, null, null, vrIfaceMTUs, null); + } + void checkAndSetRouterSourceNatIp(Account owner, CreateNetworkCmd cmd, Network network) throws InsufficientAddressCapacityException, ResourceAllocationException { String sourceNatIp = cmd.getSourceNatIP(); if (sourceNatIp == null) { diff --git a/server/src/main/java/com/cloud/network/element/VpcVirtualRouterElement.java b/server/src/main/java/com/cloud/network/element/VpcVirtualRouterElement.java index 3d613fca18e..f393ef8a129 100644 --- a/server/src/main/java/com/cloud/network/element/VpcVirtualRouterElement.java +++ b/server/src/main/java/com/cloud/network/element/VpcVirtualRouterElement.java @@ -550,7 +550,15 @@ public class VpcVirtualRouterElement extends VirtualRouterElement implements Vpc @Override public boolean reorderAclRules(Vpc vpc, List networks, List networkACLItems) { - return true; + boolean result = true; + try { + for (Network network : networks) { + result = result && applyNetworkACLs(network, networkACLItems); + } + } catch (ResourceUnavailableException ex) { + result = false; + } + return result; } @Override diff --git a/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java b/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java index e881bd7ac6e..e579eeeecd6 100644 --- a/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java +++ b/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java @@ -1747,8 +1747,9 @@ Configurable, StateListener vms = userVmJoinDao.search(scvm, null); - boolean isDhcpSupported = _ntwkSrvcDao.areServicesSupportedInNetwork(routerJoinVO.getNetworkId(), Service.Dhcp); - boolean isDnsSupported = _ntwkSrvcDao.areServicesSupportedInNetwork(routerJoinVO.getNetworkId(), Service.Dns); + Provider provider = routerJoinVO.getVpcId() != 0 ? Provider.VPCVirtualRouter : Provider.VirtualRouter; + boolean isDhcpSupported = _ntwkSrvcDao.canProviderSupportServiceInNetwork(routerJoinVO.getNetworkId(), Service.Dhcp, provider); + boolean isDnsSupported = _ntwkSrvcDao.canProviderSupportServiceInNetwork(routerJoinVO.getNetworkId(), Service.Dns, provider); for (UserVmJoinVO vm : vms) { vmsData.append("vmName=").append(vm.getName()) .append(",macAddress=").append(vm.getMacAddress()) @@ -3336,6 +3337,7 @@ Configurable, StateListener networks = _networkDao.listByAclId(lockedAcl.getId()); + if (networks.isEmpty()) { + return networkACLItem; + } + final DataCenter dc = _entityMgr.findById(DataCenter.class, vpc.getZoneId()); final NsxProviderVO nsxProvider = nsxProviderDao.findByZoneId(dc.getId()); final NetrisProviderVO netrisProvider = netrisProviderDao.findByZoneId(dc.getId()); - List networks = _networkDao.listByAclId(lockedAcl.getId()); - if (ObjectUtils.anyNotNull(nsxProvider, netrisProvider) && !networks.isEmpty()) { + boolean isVpcNetworkACLProvider = vpcManager.isProviderSupportServiceInVpc(vpc.getId(), Network.Service.NetworkACL, Network.Provider.VPCVirtualRouter); + + if (ObjectUtils.anyNotNull(nsxProvider, netrisProvider) || isVpcNetworkACLProvider) { allAclRules = getAllAclRulesSortedByNumber(lockedAcl.getId()); - Network.Provider networkProvider = nsxProvider != null ? Network.Provider.Nsx : Network.Provider.Netris; + Network.Provider networkProvider = isVpcNetworkACLProvider ? Network.Provider.VPCVirtualRouter + : (nsxProvider != null ? Network.Provider.Nsx : Network.Provider.Netris); _networkAclMgr.reorderAclRules(vpc, networks, allAclRules, networkProvider); } return networkACLItem; @@ -1061,6 +1073,111 @@ public class NetworkACLServiceImpl extends ManagerBase implements NetworkACLServ return moveRuleToTheTop(ruleBeingMoved, allRules); } + @Override + public List importNetworkACLRules(ImportNetworkACLCmd cmd) throws ResourceUnavailableException { + long aclId = cmd.getAclId(); + Map rules = cmd.getRules(); + List createdRules = new ArrayList<>(); + List errors = new ArrayList<>(); + for (Map.Entry entry : rules.entrySet()) { + try { + Map ruleMap = (Map) entry.getValue(); + NetworkACLItem item = createACLRuleFromMap(ruleMap, aclId); + createdRules.add(item); + } catch (Exception ex) { + String error = "Failed to import rule at index " + entry.getKey() + ": " + ex.getMessage(); + errors.add(error); + logger.error(error, ex); + } + } + // no rules got imported + if (createdRules.isEmpty() && !errors.isEmpty()) { + logger.error("Failed to import any ACL rules. Errors: {}", String.join("; ", errors)); + throw new CloudRuntimeException("Failed to import any ACL rules."); + } + + // apply ACL to network + if (!createdRules.isEmpty()) { + applyNetworkACL(aclId); + } + return createdRules; + } + + private NetworkACLItem createACLRuleFromMap(Map ruleMap, long aclId) { + String protocol = (String) ruleMap.get(ApiConstants.PROTOCOL); + if (protocol == null || protocol.trim().isEmpty()) { + throw new InvalidParameterValueException("Protocol is required"); + } + String action = (String) ruleMap.getOrDefault(ApiConstants.ACTION, "deny"); + String trafficType = (String) ruleMap.getOrDefault(ApiConstants.TRAFFIC_TYPE, NetworkACLItem.TrafficType.Ingress); + String forDisplay = (String) ruleMap.getOrDefault(ApiConstants.FOR_DISPLAY, "true"); + + // Create ACL rule using the service + CreateNetworkACLCmd cmd = new CreateNetworkACLCmd(); + cmd.setAclId(aclId); + cmd.setProtocol(protocol.toLowerCase()); + cmd.setAction(action.toLowerCase()); + cmd.setTrafficType(trafficType.toLowerCase()); + cmd.setDisplay(BooleanUtils.toBoolean(forDisplay)); + + // Optional parameters + if (ruleMap.containsKey(ApiConstants.CIDR_LIST)) { + Object cidrObj = ruleMap.get(ApiConstants.CIDR_LIST); + List cidrList = new ArrayList<>(); + if (cidrObj instanceof String) { + for (String cidr : ((String) cidrObj).split(",")) { + cidrList.add(cidr.trim()); + } + } else if (cidrObj instanceof List) { + cidrList.addAll((List) cidrObj); + } + cmd.setCidrList(cidrList); + } + + if (ruleMap.containsKey(ApiConstants.START_PORT)) { + cmd.setPublicStartPort(parseInt(ruleMap.get(ApiConstants.START_PORT))); + } + + if (ruleMap.containsKey(ApiConstants.END_PORT)) { + cmd.setPublicEndPort(parseInt(ruleMap.get(ApiConstants.END_PORT))); + } + + if (ruleMap.containsKey(ApiConstants.NUMBER)) { + cmd.setNumber(parseInt(ruleMap.get(ApiConstants.NUMBER))); + } + + if (ruleMap.containsKey(ApiConstants.ICMP_TYPE)) { + cmd.setIcmpType(parseInt(ruleMap.get(ApiConstants.ICMP_TYPE))); + } + + if (ruleMap.containsKey(ApiConstants.ICMP_CODE)) { + cmd.setIcmpCode(parseInt(ruleMap.get(ApiConstants.ICMP_CODE))); + } + + if (ruleMap.containsKey(ApiConstants.ACL_REASON)) { + cmd.setReason((String) ruleMap.get(ApiConstants.ACL_REASON)); + } + + return createNetworkACLItem(cmd); + } + + private Integer parseInt(Object value) { + if (value == null) { + return null; + } + if (value instanceof Integer) { + return (Integer) value; + } + if (value instanceof String) { + try { + return Integer.parseInt((String) value); + } catch (NumberFormatException e) { + throw new InvalidParameterValueException("Invalid integer value: " + value); + } + } + throw new InvalidParameterValueException("Cannot convert to integer: " + value); + } + /** * Validates the consistency of the ACL; the validation process is the following. *
    diff --git a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java index 14b58180170..6df31c6fa53 100755 --- a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java @@ -352,7 +352,6 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, private final HashMap _resourceStateAdapters = new HashMap<>(); private final HashMap> _lifeCycleListeners = new HashMap<>(); - private HypervisorType _defaultSystemVMHypervisor; private static final int ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_COOPERATION = 30; // seconds @@ -1147,8 +1146,8 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, } protected void destroyLocalStoragePoolVolumes(long poolId) { - List rootDisks = volumeDao.findByPoolId(poolId); - List dataVolumes = volumeDao.findByPoolId(poolId, Volume.Type.DATADISK); + List rootDisks = volumeDao.findNonDestroyedVolumesByPoolId(poolId); + List dataVolumes = volumeDao.findNonDestroyedVolumesByPoolId(poolId, Volume.Type.DATADISK); List volumes = new ArrayList<>(); addVolumesToList(volumes, rootDisks); @@ -1564,7 +1563,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, throw new CloudRuntimeException("There are active VMs using the host's local storage pool. Please stop all VMs on this host that use local storage."); } } else { - logger.info("Maintenance: scheduling migration of VM {} from host {}", vm, host); + logger.info("Maintenance: scheduling migration of {} from {}", vm, host); _haMgr.scheduleMigration(vm, HighAvailabilityManager.ReasonType.HostMaintenance); } } @@ -2366,7 +2365,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, List conflictingHostIds = new ArrayList<>(CollectionUtils.intersection(hostIdsToDisconnect, hostIdsUsingTheStoragePool)); if (CollectionUtils.isNotEmpty(conflictingHostIds)) { Map> hostVolumeMap = new HashMap<>(); - List volumesInPool = volumeDao.findByPoolId(poolId); + List volumesInPool = volumeDao.findNonDestroyedVolumesByPoolId(poolId); Map vmInstanceCache = new HashMap<>(); for (Long hostId : conflictingHostIds) { @@ -2935,7 +2934,6 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, @Override public boolean configure(final String name, final Map params) throws ConfigurationException { - _defaultSystemVMHypervisor = HypervisorType.getType(_configDao.getValue(Config.SystemVMDefaultHypervisor.toString())); _gson = GsonHelper.getGson(); _hypervisorsInDC = _hostDao.createSearchBuilder(String.class); @@ -2981,10 +2979,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, @Override public HypervisorType getDefaultHypervisor(final long zoneId) { - HypervisorType defaultHyper = HypervisorType.None; - if (_defaultSystemVMHypervisor != HypervisorType.None) { - defaultHyper = _defaultSystemVMHypervisor; - } + HypervisorType systemVMDefaultHypervisor = HypervisorType.getType(ResourceManager.SystemVMDefaultHypervisor.value()); final DataCenterVO dc = _dcDao.findById(zoneId); if (dc == null) { @@ -2993,27 +2988,27 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, _dcDao.loadDetails(dc); final String defaultHypervisorInZone = dc.getDetail("defaultSystemVMHypervisorType"); if (defaultHypervisorInZone != null) { - defaultHyper = HypervisorType.getType(defaultHypervisorInZone); + systemVMDefaultHypervisor = HypervisorType.getType(defaultHypervisorInZone); } final List systemTemplates = _templateDao.listAllSystemVMTemplates(); boolean isValid = false; for (final VMTemplateVO template : systemTemplates) { - if (template.getHypervisorType() == defaultHyper) { + if (template.getHypervisorType() == systemVMDefaultHypervisor) { isValid = true; break; } } if (isValid) { - final List clusters = _clusterDao.listByDcHyType(zoneId, defaultHyper.toString()); + final List clusters = _clusterDao.listByDcHyType(zoneId, systemVMDefaultHypervisor.toString()); if (clusters.isEmpty()) { isValid = false; } } if (isValid) { - return defaultHyper; + return systemVMDefaultHypervisor; } else { return HypervisorType.None; } @@ -3838,8 +3833,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, if (!isAgentOnHost || vmsMigrating || host.getStatus() == Status.Up) { return; } - final boolean sshToAgent = Boolean.parseBoolean(_configDao.getValue(KvmSshToAgentEnabled.key())); - if (sshToAgent) { + if (KvmSshToAgentEnabled.value()) { Ternary credentials = getHostCredentials(host); connectAndRestartAgentOnHost(host, credentials.first(), credentials.second(), credentials.third()); } else { @@ -4578,7 +4572,8 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, return new ConfigKey[] { KvmSshToAgentEnabled, HOST_MAINTENANCE_LOCAL_STRATEGY, - SystemVmPreferredArchitecture + SystemVmPreferredArchitecture, + SystemVMDefaultHypervisor }; } } diff --git a/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java b/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java index 648abf0d938..0da403c35f2 100644 --- a/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java +++ b/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java @@ -21,6 +21,7 @@ import static com.cloud.utils.NumbersUtil.toHumanReadableSize; import java.util.ArrayList; import java.util.Arrays; import java.util.Date; +import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -36,9 +37,6 @@ import java.util.stream.Stream; import javax.inject.Inject; import javax.naming.ConfigurationException; -import com.cloud.event.ActionEventUtils; -import com.cloud.event.EventTypes; -import com.cloud.utils.Ternary; import org.apache.cloudstack.acl.SecurityChecker.AccessType; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.response.AccountResponse; @@ -86,6 +84,8 @@ import com.cloud.dc.dao.VlanDao; import com.cloud.domain.Domain; import com.cloud.domain.DomainVO; import com.cloud.domain.dao.DomainDao; +import com.cloud.event.ActionEventUtils; +import com.cloud.event.EventTypes; import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.PermissionDeniedException; import com.cloud.exception.ResourceAllocationException; @@ -118,6 +118,7 @@ import com.cloud.user.AccountVO; import com.cloud.user.ResourceLimitService; import com.cloud.user.dao.AccountDao; import com.cloud.utils.Pair; +import com.cloud.utils.Ternary; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.concurrency.NamedThreadFactory; import com.cloud.utils.db.DB; @@ -804,45 +805,46 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim limits.addAll(foundLimits); } } else { - limits.addAll(foundLimits); - - // see if any limits are missing from the table, and if yes - get it from the config table and add - ResourceType[] resourceTypes = ResourceCount.ResourceType.values(); - if (foundLimits.size() != resourceTypes.length) { - List accountLimitStr = new ArrayList<>(); - List domainLimitStr = new ArrayList<>(); - for (ResourceLimitVO foundLimit : foundLimits) { - if (foundLimit.getAccountId() != null) { - accountLimitStr.add(foundLimit.getType().toString()); - } else { - domainLimitStr.add(foundLimit.getType().toString()); - } - } - - // get default from config values - if (isAccount) { - if (accountLimitStr.size() < resourceTypes.length) { - for (ResourceType rt : resourceTypes) { - if (!accountLimitStr.contains(rt.toString())) { - limits.add(new ResourceLimitVO(rt, findCorrectResourceLimitForAccount(_accountMgr.getAccount(accountId), rt, null), accountId, ResourceOwnerType.Account)); - } - } - } - } else { - if (domainLimitStr.size() < resourceTypes.length) { - for (ResourceType rt : resourceTypes) { - if (!domainLimitStr.contains(rt.toString())) { - limits.add(new ResourceLimitVO(rt, findCorrectResourceLimitForDomain(_domainDao.findById(domainId), rt, null), domainId, ResourceOwnerType.Domain)); - } - } - } - } - } + limits = getConsolidatedResourceLimitsForAllResourceTypes(accountId, domainId, foundLimits, isAccount); } addTaggedResourceLimits(limits, resourceType, isAccount ? ResourceOwnerType.Account : ResourceOwnerType.Domain, isAccount ? accountId : domainId, hostTags, storageTags); return limits; } + protected List getConsolidatedResourceLimitsForAllResourceTypes(Long accountId, Long domainId, + List foundLimits, boolean isAccount) { + List limits = new ArrayList<>(foundLimits); + + Set allResourceTypes = EnumSet.allOf(ResourceType.class); + Set foundUntaggedTypes = foundLimits.stream() + .filter(l -> StringUtils.isEmpty(l.getTag())) + .map(ResourceLimitVO::getType) + .collect(Collectors.toSet()); + + if (foundUntaggedTypes.containsAll(allResourceTypes)) { + return limits; + } + + ResourceOwnerType ownerType = isAccount ? ResourceOwnerType.Account : ResourceOwnerType.Domain; + long ownerId = isAccount ? accountId : domainId; + + for (ResourceType rt : allResourceTypes) { + if (foundUntaggedTypes.contains(rt)) { + continue; + } + long max; + if (isAccount) { + Account acct = _accountMgr.getAccount(accountId); + max = findCorrectResourceLimitForAccount(acct, rt, null); + } else { + DomainVO dom = _domainDao.findById(domainId); + max = findCorrectResourceLimitForDomain(dom, rt, null); + } + limits.add(new ResourceLimitVO(rt, max, ownerId, ownerType)); + } + return limits; + } + protected void addTaggedResourceLimits(List limits, List types, List tags, ResourceOwnerType ownerType, long ownerId) { if (CollectionUtils.isEmpty(tags)) { return; @@ -1222,7 +1224,6 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim } return Transaction.execute((TransactionCallback) status -> { - long newResourceCount = 0L; List domainIdList = childDomains.stream().map(DomainVO::getId).collect(Collectors.toList()); domainIdList.add(domainId); List accountIdList = accounts.stream().map(AccountVO::getId).collect(Collectors.toList()); @@ -1240,6 +1241,7 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim List resourceCounts = _resourceCountDao.lockRows(rowIdsToLock); long oldResourceCount = 0L; + long newResourceCount = 0L; ResourceCountVO domainRC = null; // calculate project count here @@ -1261,7 +1263,7 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim if (oldResourceCount != newResourceCount) { domainRC.setCount(newResourceCount); _resourceCountDao.update(domainRC.getId(), domainRC); - logger.warn("Discrepency in the resource count has been detected (original count = {} correct count = {}) for Type = {} for Domain ID = {} is fixed during resource count recalculation.", + logger.warn("Discrepancy in the resource count has been detected (original count = {} correct count = {}) for Type = {} for Domain ID = {} is fixed during resource count recalculation.", oldResourceCount, newResourceCount, type, domainId); } return newResourceCount; @@ -1524,16 +1526,17 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim } protected long calculatePrimaryStorageForAccount(long accountId, String tag) { + long snapshotsPhysicalSizeOnPrimaryStorage = _snapshotDataStoreDao.getSnapshotsPhysicalSizeOnPrimaryStorageByAccountId(accountId); if (StringUtils.isEmpty(tag)) { List virtualRouters = _vmDao.findIdsOfAllocatedVirtualRoutersForAccount(accountId); - return _volumeDao.primaryStorageUsedForAccount(accountId, virtualRouters); + return snapshotsPhysicalSizeOnPrimaryStorage + _volumeDao.primaryStorageUsedForAccount(accountId, virtualRouters); } long storage = 0; List volumes = getVolumesWithAccountAndTag(accountId, tag); for (VolumeVO volume : volumes) { storage += volume.getSize() == null ? 0L : volume.getSize(); } - return storage; + return snapshotsPhysicalSizeOnPrimaryStorage + storage; } @Override @@ -2293,7 +2296,6 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim protected class ResourceCountCheckTask extends ManagedContextRunnable { public ResourceCountCheckTask() { - } @Override diff --git a/server/src/main/java/com/cloud/server/ConfigurationServerImpl.java b/server/src/main/java/com/cloud/server/ConfigurationServerImpl.java index 51793f22e90..8f10dd84b54 100644 --- a/server/src/main/java/com/cloud/server/ConfigurationServerImpl.java +++ b/server/src/main/java/com/cloud/server/ConfigurationServerImpl.java @@ -614,7 +614,7 @@ public class ConfigurationServerImpl extends ManagerBase implements Configuratio } // FIXME: take a global database lock here for safety. boolean onWindows = isOnWindows(); - if(!onWindows) { + if (!onWindows && !(privkeyfile.exists() && pubkeyfile.exists())) { Script.runSimpleBashScript("if [ -f " + privkeyfile + " ]; then rm -f " + privkeyfile + "; fi; ssh-keygen -t ecdsa -m PEM -N '' -f " + privkeyfile + " -q 2>/dev/null || ssh-keygen -t ecdsa -N '' -f " + privkeyfile + " -q"); } diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index 76021c38278..61d5fd0fdec 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -458,6 +458,7 @@ import org.apache.cloudstack.api.command.user.network.CreateNetworkPermissionsCm import org.apache.cloudstack.api.command.user.network.DeleteNetworkACLCmd; import org.apache.cloudstack.api.command.user.network.DeleteNetworkACLListCmd; import org.apache.cloudstack.api.command.user.network.DeleteNetworkCmd; +import org.apache.cloudstack.api.command.user.network.ImportNetworkACLCmd; import org.apache.cloudstack.api.command.user.network.ListNetworkACLListsCmd; import org.apache.cloudstack.api.command.user.network.ListNetworkACLsCmd; import org.apache.cloudstack.api.command.user.network.ListNetworkOfferingsCmd; @@ -686,15 +687,15 @@ import com.cloud.alert.AlertManager; import com.cloud.alert.AlertVO; import com.cloud.alert.dao.AlertDao; import com.cloud.api.ApiDBUtils; -import com.cloud.api.query.dao.ManagementServerJoinDao; import com.cloud.api.query.dao.StoragePoolJoinDao; -import com.cloud.api.query.vo.ManagementServerJoinVO; import com.cloud.api.query.vo.StoragePoolJoinVO; import com.cloud.capacity.Capacity; import com.cloud.capacity.CapacityVO; import com.cloud.capacity.dao.CapacityDao; import com.cloud.capacity.dao.CapacityDaoImpl.SummedCapacity; import com.cloud.cluster.ClusterManager; +import com.cloud.cluster.ManagementServerHostVO; +import com.cloud.cluster.dao.ManagementServerHostDao; import com.cloud.configuration.Config; import com.cloud.configuration.ConfigurationManagerImpl; import com.cloud.consoleproxy.ConsoleProxyManagementState; @@ -796,7 +797,6 @@ import com.cloud.storage.GuestOSHypervisorVO; import com.cloud.storage.GuestOSVO; import com.cloud.storage.GuestOsCategory; import com.cloud.storage.ScopeType; -import com.cloud.storage.snapshot.SnapshotManager; import com.cloud.storage.Storage; import com.cloud.storage.StorageManager; import com.cloud.storage.StoragePool; @@ -813,6 +813,7 @@ import com.cloud.storage.dao.StoragePoolTagsDao; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.storage.secondary.SecondaryStorageVmManager; +import com.cloud.storage.snapshot.SnapshotManager; import com.cloud.tags.ResourceTagVO; import com.cloud.tags.dao.ResourceTagDao; import com.cloud.template.TemplateManager; @@ -1051,7 +1052,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe @Inject private BackupManager backupManager; @Inject - protected ManagementServerJoinDao managementServerJoinDao; + protected ManagementServerHostDao managementServerHostDao; @Inject ClusterManager _clusterMgr; @@ -4049,6 +4050,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe cmdList.add(EnableStaticNatCmd.class); cmdList.add(ListIpForwardingRulesCmd.class); cmdList.add(CreateNetworkACLCmd.class); + cmdList.add(ImportNetworkACLCmd.class); cmdList.add(CreateNetworkCmd.class); cmdList.add(DeleteNetworkACLCmd.class); cmdList.add(DeleteNetworkCmd.class); @@ -4817,6 +4819,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe final long diskOffMinSize = VolumeOrchestrationService.CustomDiskOfferingMinSize.value(); final long diskOffMaxSize = VolumeOrchestrationService.CustomDiskOfferingMaxSize.value(); final boolean KVMSnapshotEnabled = SnapshotManager.KVMSnapshotEnabled.value(); + final boolean SnapshotShowChainSize = SnapshotManager.snapshotShowChainSize.value(); final boolean userPublicTemplateEnabled = TemplateManager.AllowPublicUserTemplates.valueIn(caller.getId()); @@ -4857,6 +4860,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe capabilities.put("customDiskOffMaxSize", diskOffMaxSize); capabilities.put("regionSecondaryEnabled", regionSecondaryEnabled); capabilities.put("KVMSnapshotEnabled", KVMSnapshotEnabled); + capabilities.put("SnapshotShowChainSize", SnapshotShowChainSize); capabilities.put("allowUserViewDestroyedVM", allowUserViewDestroyedVM); capabilities.put("allowUserExpungeRecoverVM", allowUserExpungeRecoverVM); capabilities.put("allowUserExpungeRecoverVolume", allowUserExpungeRecoverVolume); @@ -5937,7 +5941,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe @ActionEvent(eventType = EventTypes.EVENT_MANAGEMENT_SERVER_REMOVE, eventDescription = "removing Management Server") public boolean removeManagementServer(RemoveManagementServerCmd cmd) { final Long id = cmd.getId(); - ManagementServerJoinVO managementServer = managementServerJoinDao.findById(id); + ManagementServerHostVO managementServer = managementServerHostDao.findById(id); if (managementServer == null) { throw new InvalidParameterValueException(String.format("Unable to find a Management Server with ID equal to [%s].", id)); @@ -5947,8 +5951,8 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe throw new InvalidParameterValueException(String.format("Unable to remove Management Server with ID [%s]. It can only be removed when it is in the [%s] state, however currently it is in the [%s] state.", managementServer.getUuid(), ManagementServerHost.State.Down.name(), managementServer.getState().name())); } - managementServer.setRemoved(new Date()); - return managementServerJoinDao.update(id, managementServer); + managementServerHostDao.remove(id); + return true; } diff --git a/server/src/main/java/com/cloud/server/StatsCollector.java b/server/src/main/java/com/cloud/server/StatsCollector.java index abc2257b00a..049ed454361 100644 --- a/server/src/main/java/com/cloud/server/StatsCollector.java +++ b/server/src/main/java/com/cloud/server/StatsCollector.java @@ -1649,7 +1649,7 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc List pools = _storagePoolDao.listAll(); for (StoragePoolVO pool : pools) { - List volumes = _volsDao.findByPoolId(pool.getId(), null); + List volumes = _volsDao.findNonDestroyedVolumesByPoolId(pool.getId(), null); for (VolumeVO volume : volumes) { if (!List.of(ImageFormat.QCOW2, ImageFormat.VHD, ImageFormat.OVA, ImageFormat.RAW).contains(volume.getFormat()) && !List.of(Storage.StoragePoolType.PowerFlex, Storage.StoragePoolType.FiberChannel).contains(pool.getPoolType())) { diff --git a/server/src/main/java/com/cloud/storage/ImageStoreDetailsUtil.java b/server/src/main/java/com/cloud/storage/ImageStoreDetailsUtil.java index baf5ef8902d..9f5aa660f4f 100755 --- a/server/src/main/java/com/cloud/storage/ImageStoreDetailsUtil.java +++ b/server/src/main/java/com/cloud/storage/ImageStoreDetailsUtil.java @@ -78,4 +78,15 @@ public class ImageStoreDetailsUtil { return getGlobalDefaultNfsVersion(); } + public boolean isCopyTemplatesFromOtherStoragesEnabled(Long storeId, Long zoneId) { + final Map storeDetails = imageStoreDetailsDao.getDetails(storeId); + final String keyWithoutDots = StorageManager.COPY_TEMPLATES_FROM_OTHER_SECONDARY_STORAGES.key() + .replace(".", ""); + + if (storeDetails != null && storeDetails.containsKey(keyWithoutDots)) { + return Boolean.parseBoolean(storeDetails.get(keyWithoutDots)); + } + + return StorageManager.COPY_TEMPLATES_FROM_OTHER_SECONDARY_STORAGES.valueIn(zoneId); + } } diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index bde6c07d1c2..1f453a86294 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -1801,14 +1801,22 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C protected String getStoragePoolNonDestroyedVolumesLog(long storagePoolId) { StringBuilder sb = new StringBuilder(); - List nonDestroyedVols = volumeDao.findByPoolId(storagePoolId, null).stream().filter(vol -> vol.getState() != Volume.State.Destroy).collect(Collectors.toList()); + List nonDestroyedVols = volumeDao.findNonDestroyedVolumesByPoolId(storagePoolId, null); VMInstanceVO volInstance; List logMessageInfo = new ArrayList<>(); sb.append("["); for (VolumeVO vol : nonDestroyedVols) { - volInstance = _vmInstanceDao.findById(vol.getInstanceId()); - logMessageInfo.add(String.format("Volume [%s] (attached to VM [%s])", vol.getUuid(), volInstance.getUuid())); + if (vol.getInstanceId() != null) { + volInstance = _vmInstanceDao.findById(vol.getInstanceId()); + if (volInstance != null) { + logMessageInfo.add(String.format("Volume [%s] (attached to VM [%s])", vol.getUuid(), volInstance.getUuid())); + } else { + logMessageInfo.add(String.format("Volume [%s] (attached VM with ID [%d] doesn't exists)", vol.getUuid(), vol.getInstanceId())); + } + } else { + logMessageInfo.add(String.format("Volume [%s] (not attached to any VM)", vol.getUuid())); + } } sb.append(String.join(", ", logMessageInfo)); sb.append("]"); @@ -3020,7 +3028,7 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C for (String childDatastoreUUID : childDatastoreUUIDs) { StoragePoolVO dataStoreVO = _storagePoolDao.findPoolByUUID(childDatastoreUUID); - List allVolumes = volumeDao.findByPoolId(dataStoreVO.getId()); + List allVolumes = volumeDao.findNonDestroyedVolumesByPoolId(dataStoreVO.getId()); allVolumes.removeIf(volumeVO -> volumeVO.getInstanceId() == null); allVolumes.removeIf(volumeVO -> volumeVO.getState() != Volume.State.Ready); for (VolumeVO volume : allVolumes) { @@ -4608,7 +4616,7 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C AllowVolumeReSizeBeyondAllocation, StoragePoolHostConnectWorkers, ObjectStorageCapacityThreshold, - COPY_PUBLIC_TEMPLATES_FROM_OTHER_STORAGES + COPY_TEMPLATES_FROM_OTHER_SECONDARY_STORAGES }; } diff --git a/server/src/main/java/com/cloud/storage/StoragePoolAutomationImpl.java b/server/src/main/java/com/cloud/storage/StoragePoolAutomationImpl.java index a5192e25a63..7121f46f486 100644 --- a/server/src/main/java/com/cloud/storage/StoragePoolAutomationImpl.java +++ b/server/src/main/java/com/cloud/storage/StoragePoolAutomationImpl.java @@ -128,7 +128,7 @@ public class StoragePoolAutomationImpl implements StoragePoolAutomation { boolean restart = !CollectionUtils.isEmpty(upPools); // 2. Get a list of all the ROOT volumes within this storage pool - List allVolumes = volumeDao.findByPoolId(pool.getId()); + List allVolumes = volumeDao.findNonDestroyedVolumesByPoolId(pool.getId()); // 3. Enqueue to the work queue enqueueMigrationsForVolumes(allVolumes, pool); // 4. Process the queue diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index ce7fab9272e..31bf80e6459 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -504,9 +504,10 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic GetUploadParamsResponse response = new GetUploadParamsResponse(); String ssvmUrlDomain = _configDao.getValue(Config.SecStorageSecureCopyCert.key()); - String protocol = UseHttpsToUpload.value() ? "https" : "http"; + String protocol = UseHttpsToUpload.valueIn(zoneId) ? "https" : "http"; - String url = ImageStoreUtil.generatePostUploadUrl(ssvmUrlDomain, ep.getPublicAddr(), vol.getUuid(), protocol); + String url = ImageStoreUtil.generatePostUploadUrl(ssvmUrlDomain, ep.getPublicAddr(), vol.getUuid(), + protocol); response.setPostURL(new URL(url)); // set the post url, this is used in the monitoring thread to determine the SSVM @@ -526,8 +527,10 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic /* * encoded metadata using the post upload config key */ - TemplateOrVolumePostUploadCommand command = new TemplateOrVolumePostUploadCommand(vol.getId(), vol.getUuid(), volumeStore.getInstallPath(), cmd.getChecksum(), vol.getType().toString(), - vol.getName(), vol.getFormat().toString(), dataObject.getDataStore().getUri(), dataObject.getDataStore().getRole().toString()); + TemplateOrVolumePostUploadCommand command = new TemplateOrVolumePostUploadCommand(vol.getId(), + vol.getUuid(), volumeStore.getInstallPath(), cmd.getChecksum(), vol.getType().toString(), + vol.getName(), vol.getFormat().toString(), dataObject.getDataStore().getUri(), + dataObject.getDataStore().getRole().toString(), zoneId); command.setLocalPath(volumeStore.getLocalDownloadPath()); //using the existing max upload size configuration command.setProcessTimeout(NumbersUtil.parseLong(_configDao.getValue("vmware.package.ova.timeout"), 3600)); diff --git a/server/src/main/java/com/cloud/storage/listener/StoragePoolMonitor.java b/server/src/main/java/com/cloud/storage/listener/StoragePoolMonitor.java index b432858f2e0..2f9750edee2 100644 --- a/server/src/main/java/com/cloud/storage/listener/StoragePoolMonitor.java +++ b/server/src/main/java/com/cloud/storage/listener/StoragePoolMonitor.java @@ -27,6 +27,7 @@ import com.cloud.exception.StorageConflictException; import com.cloud.storage.StorageManager; import com.cloud.storage.dao.StoragePoolHostDao; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.Profiler; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProvider; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProviderManager; import org.apache.cloudstack.engine.subsystem.api.storage.HypervisorHostListener; @@ -200,12 +201,13 @@ public class StoragePoolMonitor implements Listener { } @Override - public synchronized boolean processDisconnect(long agentId, Status state) { + public boolean processDisconnect(long agentId, Status state) { return processDisconnect(agentId, null, null, state); } @Override - public synchronized boolean processDisconnect(long agentId, String uuid, String name, Status state) { + public boolean processDisconnect(long agentId, String uuid, String name, Status state) { + logger.debug("Starting disconnect for Agent [id: {}, uuid: {}, name: {}]", agentId, uuid, name); Host host = _storageManager.getHost(agentId); if (host == null) { logger.warn("Agent [id: {}, uuid: {}, name: {}] not found, not disconnecting pools", agentId, uuid, name); @@ -213,38 +215,52 @@ public class StoragePoolMonitor implements Listener { } if (host.getType() != Host.Type.Routing) { + logger.debug("Host [id: {}, uuid: {}, name: {}] is not of type {}, skip", agentId, uuid, name, Host.Type.Routing); return false; } + logger.debug("Looking for connected Storage Pools for Host [id: {}, uuid: {}, name: {}]", agentId, uuid, name); List storagePoolHosts = _storageManager.findStoragePoolsConnectedToHost(host.getId()); if (storagePoolHosts == null) { - if (logger.isTraceEnabled()) { - logger.trace("No pools to disconnect for host: {}", host); - } + logger.debug("No pools to disconnect for host: {}", host); return true; } + logger.debug("Found {} pools to disconnect for host: {}", storagePoolHosts.size(), host); boolean disconnectResult = true; - for (StoragePoolHostVO storagePoolHost : storagePoolHosts) { + int storagePoolHostsSize = storagePoolHosts.size(); + for (int i = 0; i < storagePoolHostsSize; i++) { + StoragePoolHostVO storagePoolHost = storagePoolHosts.get(i); + logger.debug("Processing disconnect from Storage Pool {} ({} of {}) for host: {}", storagePoolHost.getPoolId(), i, storagePoolHostsSize, host); StoragePoolVO pool = _poolDao.findById(storagePoolHost.getPoolId()); if (pool == null) { + logger.debug("No Storage Pool found with id {} ({} of {}) for host: {}", storagePoolHost.getPoolId(), i, storagePoolHostsSize, host); continue; } if (!pool.isShared()) { + logger.debug("Storage Pool {} ({}) ({} of {}) is not shared for host: {}, ignore disconnect", pool.getName(), pool.getUuid(), i, storagePoolHostsSize, host); continue; } // Handle only PowerFlex pool for now, not to impact other pools behavior if (pool.getPoolType() != StoragePoolType.PowerFlex) { + logger.debug("Storage Pool {} ({}) ({} of {}) is not of type {} for host: {}, ignore disconnect", pool.getName(), pool.getUuid(), i, storagePoolHostsSize, pool.getPoolType(), host); continue; } + logger.debug("Sending disconnect to Storage Pool {} ({}) ({} of {}) for host: {}", pool.getName(), pool.getUuid(), i, storagePoolHostsSize, host); + Profiler disconnectProfiler = new Profiler(); try { + disconnectProfiler.start(); _storageManager.disconnectHostFromSharedPool(host, pool); } catch (Exception e) { logger.error("Unable to disconnect host {} from storage pool {} due to {}", host, pool, e.toString()); disconnectResult = false; + } finally { + disconnectProfiler.stop(); + long disconnectDuration = disconnectProfiler.getDurationInMillis() / 1000; + logger.debug("Finished disconnect with result {} from Storage Pool {} ({}) ({} of {}) for host: {}, duration: {} secs", disconnectResult, pool.getName(), pool.getUuid(), i, storagePoolHostsSize, host, disconnectDuration); } } diff --git a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java index ff9989acac3..b5a03d1836b 100755 --- a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java @@ -290,6 +290,15 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement return !DataCenter.Type.Edge.equals(zone.getType()); } + private ResourceType getStoreResourceType(long dataCenterId, Snapshot.LocationType locationType) { + ResourceType storeResourceType = ResourceType.secondary_storage; + if (!isBackupSnapshotToSecondaryForZone(dataCenterId) || + Snapshot.LocationType.PRIMARY.equals(locationType)) { + storeResourceType = ResourceType.primary_storage; + } + return storeResourceType; + } + @Override public String getConfigComponentName() { return SnapshotManager.class.getSimpleName(); @@ -578,8 +587,9 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement } if (ObjectUtils.anyNull(chosenStore, snapshotDataStoreReference)) { - logger.error("Snapshot [{}] not found in any secondary storage.", snapshot); - throw new InvalidParameterValueException("Snapshot not found."); + String errorMessage = String.format("Snapshot [%s] not found in any secondary storage. The snapshot may be on primary storage, where it cannot be downloaded.", snapshot.getUuid()); + logger.error(errorMessage); + throw new InvalidParameterValueException(errorMessage); } snapshotSrv.syncVolumeSnapshotsToRegionStore(snapshot.getVolumeId(), chosenStore); @@ -675,7 +685,7 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement _snapshotDao.update(snapshot.getId(), snapshot); snapshotInfo = this.snapshotFactory.getSnapshot(snapshotId, store); - Long snapshotOwnerId = vm.getAccountId(); + long snapshotOwnerId = vm.getAccountId(); try { SnapshotStrategy snapshotStrategy = _storageStrategyFactory.getSnapshotStrategy(snapshot, SnapshotOperation.BACKUP); @@ -683,7 +693,6 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement throw new CloudRuntimeException(String.format("Unable to find Snapshot strategy to handle Snapshot [%s]", snapshot)); } snapshotInfo = snapshotStrategy.backupSnapshot(snapshotInfo); - } catch (Exception e) { logger.debug("Failed to backup Snapshot from Instance Snapshot", e); _resourceLimitMgr.decrementResourceCount(snapshotOwnerId, ResourceType.snapshot); @@ -893,12 +902,11 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement _accountMgr.checkAccess(caller, null, true, snapshotCheck); SnapshotStrategy snapshotStrategy = _storageStrategyFactory.getSnapshotStrategy(snapshotCheck, zoneId, SnapshotOperation.DELETE); - if (snapshotStrategy == null) { logger.error("Unable to find snapshot strategy to handle snapshot [{}]", snapshotCheck); - return false; } + Pair, List> storeRefAndZones = getStoreRefsAndZonesForSnapshotDelete(snapshotId, zoneId); List snapshotStoreRefs = storeRefAndZones.first(); List zoneIds = storeRefAndZones.second(); @@ -1688,8 +1696,9 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement UsageEventUtils.publishUsageEvent(EventTypes.EVENT_SNAPSHOT_CREATE, snapshot.getAccountId(), snapshot.getDataCenterId(), snapshotId, snapshot.getName(), null, null, snapshotStoreRef.getPhysicalSize(), volume.getSize(), snapshot.getClass().getName(), snapshot.getUuid()); + ResourceType storeResourceType = dataStoreRole == DataStoreRole.Image ? ResourceType.secondary_storage : ResourceType.primary_storage; // Correct the resource count of snapshot in case of delta snapshots. - _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), ResourceType.secondary_storage, new Long(volume.getSize() - snapshotStoreRef.getPhysicalSize())); + _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), storeResourceType, new Long(volume.getSize() - snapshotStoreRef.getPhysicalSize())); if (!payload.getAsyncBackup()) { if (backupSnapToSecondary) { @@ -1706,15 +1715,17 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement if (logger.isDebugEnabled()) { logger.debug("Failed to create snapshot" + cre.getLocalizedMessage()); } + ResourceType storeResourceType = getStoreResourceType(volume.getDataCenterId(), payload.getLocationType()); _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), ResourceType.snapshot); - _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), ResourceType.secondary_storage, new Long(volume.getSize())); + _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), storeResourceType, new Long(volume.getSize())); throw cre; } catch (Exception e) { if (logger.isDebugEnabled()) { logger.debug("Failed to create snapshot", e); } + ResourceType storeResourceType = getStoreResourceType(volume.getDataCenterId(), payload.getLocationType()); _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), ResourceType.snapshot); - _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), ResourceType.secondary_storage, new Long(volume.getSize())); + _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), storeResourceType, new Long(volume.getSize())); throw new CloudRuntimeException("Failed to create snapshot", e); } return snapshot; @@ -1889,9 +1900,25 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement logger.debug("Failed to delete snapshot in destroying state: {}", snapshotVO); } } + cleanupOrphanSnapshotPolicies(); + return true; } + private void cleanupOrphanSnapshotPolicies() { + List policies = _snapshotPolicyDao.listActivePolicies(); + if (CollectionUtils.isEmpty(policies)) { + return; + } + for (SnapshotPolicyVO policy : policies) { + VolumeVO volume = _volsDao.findByIdIncludingRemoved(policy.getVolumeId()); + if (volume == null || volume.getState() == Volume.State.Expunged) { + logger.info("Removing orphan snapshot policy {} for non-existent volume {}", policy.getId(), policy.getVolumeId()); + deletePolicy(policy.getId()); + } + } + } + @Override public boolean stop() { backupSnapshotExecutor.shutdown(); @@ -1924,7 +1951,7 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement if (snapshotPolicyVO == null) { throw new InvalidParameterValueException("Policy id given: " + policy + " does not exist"); } - VolumeVO volume = _volsDao.findById(snapshotPolicyVO.getVolumeId()); + VolumeVO volume = _volsDao.findByIdIncludingRemoved(snapshotPolicyVO.getVolumeId()); if (volume == null) { throw new InvalidParameterValueException("Policy id given: " + policy + " does not belong to a valid volume"); } @@ -1991,11 +2018,7 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement Type snapshotType = getSnapshotType(policyId); Account owner = _accountMgr.getAccount(volume.getAccountId()); - ResourceType storeResourceType = ResourceType.secondary_storage; - if (!isBackupSnapshotToSecondaryForZone(volume.getDataCenterId()) || - Snapshot.LocationType.PRIMARY.equals(locationType)) { - storeResourceType = ResourceType.primary_storage; - } + ResourceType storeResourceType = getStoreResourceType(volume.getDataCenterId(), locationType); try { _resourceLimitMgr.checkResourceLimit(owner, ResourceType.snapshot); _resourceLimitMgr.checkResourceLimit(owner, storeResourceType, volume.getSize()); diff --git a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java index bc3d219395f..2e4deafe576 100644 --- a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java +++ b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java @@ -288,7 +288,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { for (long zoneId : zonesIds) { - DataStore imageStore = verifyHeuristicRulesForZone(template, zoneId); + DataStore imageStore = templateMgr.verifyHeuristicRulesForZone(template, zoneId); if (imageStore == null) { List imageStores = getImageStoresThrowsExceptionIfNotFound(zoneId, profile); @@ -299,6 +299,14 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { } } + protected List getImageStoresThrowsExceptionIfNotFound(long zoneId, TemplateProfile profile) { + List imageStores = storeMgr.getImageStoresByZoneIds(zoneId); + if (imageStores == null || imageStores.size() == 0) { + throw new CloudRuntimeException(String.format("Unable to find image store to download the template [%s].", profile.getTemplate())); + } + return imageStores; + } + protected void standardImageStoreAllocation(List imageStores, VMTemplateVO template) { Set zoneSet = new HashSet(); Collections.shuffle(imageStores); @@ -356,7 +364,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { } Long zoneId = zoneIdList.get(0); - DataStore imageStore = verifyHeuristicRulesForZone(template, zoneId); + DataStore imageStore = templateMgr.verifyHeuristicRulesForZone(template, zoneId); List payloads = new LinkedList<>(); if (imageStore == null) { diff --git a/server/src/main/java/com/cloud/template/TemplateAdapterBase.java b/server/src/main/java/com/cloud/template/TemplateAdapterBase.java index da620a3375a..b50acedddc0 100644 --- a/server/src/main/java/com/cloud/template/TemplateAdapterBase.java +++ b/server/src/main/java/com/cloud/template/TemplateAdapterBase.java @@ -234,9 +234,10 @@ public abstract class TemplateAdapterBase extends AdapterBase implements Templat throw new CloudRuntimeException(errMsg); } - TemplateOrVolumePostUploadCommand payload = new TemplateOrVolumePostUploadCommand(template.getId(), template.getUuid(), tmpl.getInstallPath(), tmpl - .getChecksum(), tmpl.getType().toString(), template.getUniqueName(), template.getFormat().toString(), templateOnStore.getDataStore().getUri(), - templateOnStore.getDataStore().getRole().toString()); + TemplateOrVolumePostUploadCommand payload = new TemplateOrVolumePostUploadCommand(template.getId(), + template.getUuid(), tmpl.getInstallPath(), tmpl.getChecksum(), tmpl.getType().toString(), + template.getUniqueName(), template.getFormat().toString(), templateOnStore.getDataStore().getUri(), + templateOnStore.getDataStore().getRole().toString(), zoneId_is); //using the existing max template size configuration payload.setMaxUploadSize(_configDao.getValue(Config.MaxTemplateAndIsoSize.key())); diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index a37be2e73db..0b05a0d9c3d 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -121,6 +121,7 @@ import com.cloud.agent.api.to.DatadiskTO; import com.cloud.agent.api.to.DiskTO; import com.cloud.agent.api.to.NfsTO; import com.cloud.agent.api.to.VirtualMachineTO; +import com.cloud.agent.api.to.deployasis.OVFNetworkTO; import com.cloud.api.ApiDBUtils; import com.cloud.api.query.dao.UserVmJoinDao; import com.cloud.api.query.vo.UserVmJoinVO; @@ -131,6 +132,7 @@ import com.cloud.dc.DataCenter; import com.cloud.dc.DataCenterVO; import com.cloud.dc.dao.DataCenterDao; import com.cloud.deploy.DeployDestination; +import com.cloud.deployasis.dao.TemplateDeployAsIsDetailsDao; import com.cloud.domain.Domain; import com.cloud.domain.dao.DomainDao; import com.cloud.event.ActionEvent; @@ -315,6 +317,8 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, protected SnapshotHelper snapshotHelper; @Inject VnfTemplateManager vnfTemplateManager; + @Inject + TemplateDeployAsIsDetailsDao templateDeployAsIsDetailsDao; @Inject private SecondaryStorageHeuristicDao secondaryStorageHeuristicDao; @@ -411,7 +415,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, TemplateOrVolumePostUploadCommand firstCommand = payload.get(0); String ssvmUrlDomain = _configDao.getValue(Config.SecStorageSecureCopyCert.key()); - String protocol = VolumeApiService.UseHttpsToUpload.value() ? "https" : "http"; + String protocol = VolumeApiService.UseHttpsToUpload.valueIn(firstCommand.getZoneId()) ? "https" : "http"; String url = ImageStoreUtil.generatePostUploadUrl(ssvmUrlDomain, firstCommand.getRemoteEndPoint(), firstCommand.getEntityUUID(), protocol); response.setPostURL(new URL(url)); @@ -844,6 +848,9 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, // Copy will just find one eligible image store for the destination zone // and copy template there, not propagate to all image stores // for that zone + + boolean copied = false; + for (DataStore dstSecStore : dstSecStores) { TemplateDataStoreVO dstTmpltStore = _tmplStoreDao.findByStoreTemplate(dstSecStore.getId(), tmpltId); if (dstTmpltStore != null && dstTmpltStore.getDownloadState() == Status.DOWNLOADED) { @@ -858,9 +865,12 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, TemplateApiResult result = future.get(); if (result.isFailed()) { logger.debug("Copy Template failed for image store {}: {}", dstSecStore, result.getResult()); + _tmplStoreDao.removeByTemplateStore(tmpltId, dstSecStore.getId()); continue; // try next image store } + copied = true; + _tmpltDao.addTemplateToZone(template, dstZoneId); if (account.getId() != Account.ACCOUNT_ID_SYSTEM) { @@ -888,12 +898,14 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, } } } + + return true; + } catch (Exception ex) { - logger.debug("Failed to copy Template to image store:{} ,will try next one", dstSecStore); + logger.debug("Failed to copy Template to image store:{} ,will try next one", dstSecStore, ex); } } - return true; - + return copied; } @Override @@ -2217,6 +2229,11 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, templateType = validateTemplateType(cmd, isAdmin, template.isCrossZones(), template.getHypervisorType()); if (cmd instanceof UpdateVnfTemplateCmd) { VnfTemplateUtils.validateApiCommandParams(cmd, template); + UpdateVnfTemplateCmd updateCmd = (UpdateVnfTemplateCmd) cmd; + if (template.isDeployAsIs() && CollectionUtils.isNotEmpty(updateCmd.getVnfNics())) { + List ovfNetworks = templateDeployAsIsDetailsDao.listNetworkRequirementsByTemplateId(template.getId()); + VnfTemplateUtils.validateDeployAsIsTemplateVnfNics(ovfNetworks, updateCmd.getVnfNics()); + } vnfTemplateManager.updateVnfTemplate(template.getId(), (UpdateVnfTemplateCmd) cmd); } templateTag = ((UpdateTemplateCmd)cmd).getTemplateTag(); @@ -2415,6 +2432,17 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, TemplateType.USER, HypervisorType.External)); } + @Override + public DataStore verifyHeuristicRulesForZone(VMTemplateVO template, Long zoneId) { + HeuristicType heuristicType; + if (ImageFormat.ISO.equals(template.getFormat())) { + heuristicType = HeuristicType.ISO; + } else { + heuristicType = HeuristicType.TEMPLATE; + } + return heuristicRuleHelper.getImageStoreIfThereIsHeuristicRule(zoneId, heuristicType, template); + } + void validateDetails(VMTemplateVO template, Map details) { if (MapUtils.isEmpty(details)) { return; diff --git a/server/src/main/java/com/cloud/usage/UsageServiceImpl.java b/server/src/main/java/com/cloud/usage/UsageServiceImpl.java index c46f1f43fd0..de8d4633d22 100644 --- a/server/src/main/java/com/cloud/usage/UsageServiceImpl.java +++ b/server/src/main/java/com/cloud/usage/UsageServiceImpl.java @@ -17,7 +17,6 @@ package com.cloud.usage; import java.util.ArrayList; -import java.util.Calendar; import java.util.Date; import java.util.List; import java.util.Map; @@ -26,6 +25,7 @@ import java.util.TimeZone; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.configuration.ConfigurationManagerImpl; import org.apache.cloudstack.api.command.admin.usage.GenerateUsageRecordsCmd; import org.apache.cloudstack.api.command.admin.usage.ListUsageRecordsCmd; import org.apache.cloudstack.api.command.admin.usage.RemoveRawUsageRecordsCmd; @@ -34,6 +34,7 @@ import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.usage.Usage; import org.apache.cloudstack.usage.UsageService; import org.apache.cloudstack.usage.UsageTypes; +import org.apache.cloudstack.utils.usage.UsageUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.jetbrains.annotations.NotNull; @@ -126,14 +127,25 @@ public class UsageServiceImpl extends ManagerBase implements UsageService, Manag @Inject private NetworkOfferingDao _networkOfferingDao; + private TimeZone usageExecutionTimeZone = TimeZone.getTimeZone("GMT"); + + private static final long REMOVE_RAW_USAGE_RECORDS_WINDOW_IN_MS = 15 * 60 * 1000; + public UsageServiceImpl() { } @Override public boolean configure(String name, Map params) throws ConfigurationException { super.configure(name, params); + String timeZoneStr = ObjectUtils.defaultIfNull(_configDao.getValue(Config.UsageAggregationTimezone.toString()), "GMT"); _usageTimezone = TimeZone.getTimeZone(timeZoneStr); + + String executionTimeZone = _configDao.getValue(Config.UsageExecutionTimezone.toString()); + if (executionTimeZone != null) { + usageExecutionTimeZone = TimeZone.getTimeZone(executionTimeZone); + } + return true; } @@ -464,35 +476,28 @@ public class UsageServiceImpl extends ManagerBase implements UsageService, Manag @Override public boolean removeRawUsageRecords(RemoveRawUsageRecordsCmd cmd) throws InvalidParameterValueException { Integer interval = cmd.getInterval(); - if (interval != null && interval > 0 ) { - String jobExecTime = _configDao.getValue(Config.UsageStatsJobExecTime.toString()); - if (jobExecTime != null ) { - String[] segments = jobExecTime.split(":"); - if (segments.length == 2) { - String timeZoneStr = _configDao.getValue(Config.UsageExecutionTimezone.toString()); - if (timeZoneStr == null) { - timeZoneStr = "GMT"; - } - TimeZone tz = TimeZone.getTimeZone(timeZoneStr); - Calendar cal = Calendar.getInstance(tz); - cal.setTime(new Date()); - long curTS = cal.getTimeInMillis(); - cal.set(Calendar.HOUR_OF_DAY, Integer.parseInt(segments[0])); - cal.set(Calendar.MINUTE, Integer.parseInt(segments[1])); - cal.set(Calendar.SECOND, 0); - cal.set(Calendar.MILLISECOND, 0); - long execTS = cal.getTimeInMillis(); - logger.debug("Trying to remove old raw cloud_usage records older than " + interval + " day(s), current time=" + curTS + " next job execution time=" + execTS); - // Let's avoid cleanup when job runs and around a 15 min interval - if (Math.abs(curTS - execTS) < 15 * 60 * 1000) { - return false; - } - } - } - _usageDao.removeOldUsageRecords(interval); - } else { - throw new InvalidParameterValueException("Invalid interval value. Interval to remove cloud_usage records should be greater than 0"); + if (interval == null || interval <= 0) { + throw new InvalidParameterValueException("Interval should be greater than 0."); } + + String jobExecTime = _configDao.getValue(Config.UsageStatsJobExecTime.toString()); + Date previousJobExecTime = UsageUtils.getPreviousJobExecutionTime(usageExecutionTimeZone, jobExecTime); + Date nextJobExecTime = UsageUtils.getNextJobExecutionTime(usageExecutionTimeZone, jobExecTime); + if (ObjectUtils.allNotNull(previousJobExecTime, nextJobExecTime)) { + logger.debug("Next Usage job is scheduled to execute at [{}]; previous execution was at [{}].", + DateUtil.displayDateInTimezone(usageExecutionTimeZone, nextJobExecTime), DateUtil.displayDateInTimezone(usageExecutionTimeZone, previousJobExecTime)); + Date now = new Date(); + if (nextJobExecTime.getTime() - now.getTime() < REMOVE_RAW_USAGE_RECORDS_WINDOW_IN_MS) { + logger.info("Not removing any cloud_usage records because the next Usage job is scheduled to execute in less than {} minute(s).", REMOVE_RAW_USAGE_RECORDS_WINDOW_IN_MS / 60000); + return false; + } else if (now.getTime() - previousJobExecTime.getTime() < REMOVE_RAW_USAGE_RECORDS_WINDOW_IN_MS) { + logger.info("Not removing any cloud_usage records because the last Usage job executed in less than {} minute(s) ago.", REMOVE_RAW_USAGE_RECORDS_WINDOW_IN_MS / 60000); + return false; + } + } + + logger.info("Removing cloud_usage records older than {} day(s).", interval); + _usageDao.expungeAllOlderThan(interval, ConfigurationManagerImpl.DELETE_QUERY_BATCH_SIZE.value()); return true; } } diff --git a/server/src/main/java/com/cloud/user/AccountManagerImpl.java b/server/src/main/java/com/cloud/user/AccountManagerImpl.java index 6f68a104867..53b88690654 100644 --- a/server/src/main/java/com/cloud/user/AccountManagerImpl.java +++ b/server/src/main/java/com/cloud/user/AccountManagerImpl.java @@ -3462,7 +3462,7 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M } @Override - public Long finalyzeAccountId(final String accountName, final Long domainId, final Long projectId, final boolean enabledOnly) { + public Long finalizeAccountId(final String accountName, final Long domainId, final Long projectId, final boolean enabledOnly) { if (accountName != null) { if (domainId == null) { throw new InvalidParameterValueException("Account must be specified with domainId parameter"); diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 17a893c4400..b9b6e5a6839 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -2309,7 +2309,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir private List getVolumesByHost(HostVO host, StoragePool pool){ List vmsPerHost = _vmInstanceDao.listByHostId(host.getId()); return vmsPerHost.stream() - .flatMap(vm -> _volsDao.findByInstanceIdAndPoolId(vm.getId(),pool.getId()).stream().map(vol -> + .flatMap(vm -> _volsDao.findNonDestroyedVolumesByInstanceIdAndPoolId(vm.getId(),pool.getId()).stream().map(vol -> vol.getState() == Volume.State.Ready ? (vol.getFormat() == ImageFormat.OVA ? vol.getChainInfo() : vol.getPath()) : null).filter(Objects::nonNull)) .collect(Collectors.toList()); } @@ -2866,6 +2866,22 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir } } + protected void updateVmExtraConfig(UserVmVO userVm, String extraConfig, boolean cleanupExtraConfig) { + if (cleanupExtraConfig) { + logger.info("Cleaning up extraconfig from user vm: {}", userVm.getUuid()); + vmInstanceDetailsDao.removeDetailsWithPrefix(userVm.getId(), ApiConstants.EXTRA_CONFIG); + return; + } + if (StringUtils.isNotBlank(extraConfig)) { + if (EnableAdditionalVmConfig.valueIn(userVm.getAccountId())) { + logger.info("Adding extra configuration to user vm: {}", userVm.getUuid()); + addExtraConfig(userVm, extraConfig); + } else { + throw new InvalidParameterValueException("attempted setting extraconfig but enable.additional.vm.configuration is disabled"); + } + } + } + @Override @ActionEvent(eventType = EventTypes.EVENT_VM_UPDATE, eventDescription = "updating Vm") public UserVm updateVirtualMachine(UpdateVMCmd cmd) throws ResourceUnavailableException, InsufficientCapacityException { @@ -2883,6 +2899,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir List securityGroupIdList = getSecurityGroupIdList(cmd); boolean cleanupDetails = cmd.isCleanupDetails(); String extraConfig = cmd.getExtraConfig(); + boolean cleanupExtraConfig = cmd.isCleanupExtraConfig(); UserVmVO vmInstance = _vmDao.findById(cmd.getId()); VMTemplateVO template = _templateDao.findById(vmInstance.getTemplateId()); @@ -2919,7 +2936,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir .map(item -> (item).trim()) .collect(Collectors.toList()); List existingDetails = vmInstanceDetailsDao.listDetails(id); - if (cleanupDetails){ + if (cleanupDetails) { if (caller != null && caller.getType() == Account.Type.ADMIN) { for (final VMInstanceDetailVO detail : existingDetails) { if (detail != null && detail.isDisplay() && !isExtraConfig(detail.getName())) { @@ -2982,15 +2999,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir vmInstance.setDetails(details); _vmDao.saveDetails(vmInstance); } - if (StringUtils.isNotBlank(extraConfig)) { - if (EnableAdditionalVmConfig.valueIn(accountId)) { - logger.info("Adding extra configuration to user vm: " + vmInstance.getUuid()); - addExtraConfig(vmInstance, extraConfig); - } else { - throw new InvalidParameterValueException("attempted setting extraconfig but enable.additional.vm.configuration is disabled"); - } - } } + updateVmExtraConfig(userVm, extraConfig, cleanupExtraConfig); if (VMLeaseManager.InstanceLeaseEnabled.value() && cmd.getLeaseDuration() != null) { applyLeaseOnUpdateInstance(vmInstance, cmd.getLeaseDuration(), cmd.getLeaseExpiryAction()); @@ -6284,7 +6294,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir private void verifyTemplate(BaseDeployVMCmd cmd, VirtualMachineTemplate template, Long serviceOfferingId) { if (TemplateType.VNF.equals(template.getTemplateType())) { - vnfTemplateManager.validateVnfApplianceNics(template, cmd.getNetworkIds()); + vnfTemplateManager.validateVnfApplianceNics(template, cmd.getNetworkIds(), cmd.getVmNetworkMap()); } else if (cmd instanceof DeployVnfApplianceCmd) { throw new InvalidParameterValueException("Can't deploy VNF appliance from a non-VNF template"); } diff --git a/server/src/main/java/com/cloud/vm/snapshot/VMSnapshotManagerImpl.java b/server/src/main/java/com/cloud/vm/snapshot/VMSnapshotManagerImpl.java index 617a4e54a6e..d5e25adcd5f 100644 --- a/server/src/main/java/com/cloud/vm/snapshot/VMSnapshotManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/snapshot/VMSnapshotManagerImpl.java @@ -400,10 +400,11 @@ public class VMSnapshotManagerImpl extends MutualExclusiveIdsManagerBase impleme _accountMgr.checkAccess(caller, null, true, userVmVo); // check max snapshot limit for per VM - int vmSnapshotMax = VMSnapshotManager.VMSnapshotMax.value(); - + boolean vmBelongsToProject = _accountMgr.getAccount(userVmVo.getAccountId()).getType() == Account.Type.PROJECT; + long accountIdToRetrieveConfigurationValueFrom = vmBelongsToProject ? caller.getId() : userVmVo.getAccountId(); + int vmSnapshotMax = VMSnapshotManager.VMSnapshotMax.valueIn(accountIdToRetrieveConfigurationValueFrom); if (_vmSnapshotDao.findByVm(vmId).size() >= vmSnapshotMax) { - throw new CloudRuntimeException("Creating Instance Snapshot failed due to a Instance can just have : " + vmSnapshotMax + " Instance Snapshots. Please delete old ones"); + throw new CloudRuntimeException(String.format("Each VM can have at most [%s] VM snapshots.", vmSnapshotMax)); } // check if there are active volume snapshots tasks diff --git a/server/src/main/java/org/apache/cloudstack/affinity/AffinityGroupServiceImpl.java b/server/src/main/java/org/apache/cloudstack/affinity/AffinityGroupServiceImpl.java index 650d52f9dc5..d212e7435b2 100644 --- a/server/src/main/java/org/apache/cloudstack/affinity/AffinityGroupServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/affinity/AffinityGroupServiceImpl.java @@ -286,7 +286,7 @@ public class AffinityGroupServiceImpl extends ManagerBase implements AffinityGro if(account == null && domainId != null){ group = _affinityGroupDao.findDomainLevelGroupByName(domainId, affinityGroupName); }else{ - Long accountId = _accountMgr.finalyzeAccountId(account, domainId, projectId, true); + Long accountId = _accountMgr.finalizeAccountId(account, domainId, projectId, true); if(accountId == null){ Account caller = CallContext.current().getCallingAccount(); group = _affinityGroupDao.findByAccountAndName(caller.getAccountId(), affinityGroupName); diff --git a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java index b83c1023662..b4433fb7540 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java @@ -426,7 +426,7 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { final Filter searchFilter = new Filter(BackupOfferingVO.class, "id", true, cmd.getStartIndex(), cmd.getPageSizeVal()); SearchBuilder sb = backupOfferingDao.createSearchBuilder(); sb.and("zone_id", sb.entity().getZoneId(), SearchCriteria.Op.EQ); - sb.and("name", sb.entity().getName(), SearchCriteria.Op.EQ); + sb.and("name", sb.entity().getName(), SearchCriteria.Op.LIKE); CallContext ctx = CallContext.current(); final Account caller = ctx.getCallingAccount(); diff --git a/server/src/main/java/org/apache/cloudstack/network/RoutedIpv4ManagerImpl.java b/server/src/main/java/org/apache/cloudstack/network/RoutedIpv4ManagerImpl.java index ac12dc3a9b1..5e49872c64d 100644 --- a/server/src/main/java/org/apache/cloudstack/network/RoutedIpv4ManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/network/RoutedIpv4ManagerImpl.java @@ -229,7 +229,7 @@ public class RoutedIpv4ManagerImpl extends ComponentLifecycleBase implements Rou Long accountId = null; if (accountName != null || (projectId != null && projectId != -1L)) { - accountId = accountManager.finalyzeAccountId(accountName, domainId, projectId, false); + accountId = accountManager.finalizeAccountId(accountName, domainId, projectId, false); } if (accountId != null) { Account account = accountManager.getAccount(accountId); @@ -371,7 +371,7 @@ public class RoutedIpv4ManagerImpl extends ComponentLifecycleBase implements Rou sc.addAnd("domainId", SearchCriteria.Op.EQ, domainId); } if (accountName != null || (projectId != null && projectId != -1L)) { - Long accountId= accountManager.finalyzeAccountId(accountName, domainId, projectId, false); + Long accountId= accountManager.finalizeAccountId(accountName, domainId, projectId, false); sc.addAnd("accountId", SearchCriteria.Op.EQ, accountId); } // search via dataCenterIpv4GuestSubnetDao @@ -394,7 +394,7 @@ public class RoutedIpv4ManagerImpl extends ComponentLifecycleBase implements Rou } Long accountId = null; if (accountName != null || (projectId != null && projectId != -1L)) { - accountId = accountManager.finalyzeAccountId(accountName, domainId, projectId, false); + accountId = accountManager.finalizeAccountId(accountName, domainId, projectId, false); } if (accountId != null) { Account account = accountManager.getAccount(accountId); @@ -1094,7 +1094,7 @@ public class RoutedIpv4ManagerImpl extends ComponentLifecycleBase implements Rou Long accountId = null; if (accountName != null || (projectId != null && projectId != -1L)) { - accountId = accountManager.finalyzeAccountId(accountName, domainId, projectId, false); + accountId = accountManager.finalizeAccountId(accountName, domainId, projectId, false); } if (accountId != null) { Account account = accountManager.getAccount(accountId); @@ -1283,7 +1283,7 @@ public class RoutedIpv4ManagerImpl extends ComponentLifecycleBase implements Rou } Long accountId = null; if (accountName != null || (projectId != null && projectId != -1L)) { - accountId = accountManager.finalyzeAccountId(accountName, domainId, projectId, false); + accountId = accountManager.finalizeAccountId(accountName, domainId, projectId, false); } if (accountId != null) { Account account = accountManager.getAccount(accountId); @@ -1350,7 +1350,7 @@ public class RoutedIpv4ManagerImpl extends ComponentLifecycleBase implements Rou Long accountId = null; if (accountName != null || (projectId != null && projectId != -1L)) { - accountId = accountManager.finalyzeAccountId(accountName, domainId, projectId, false); + accountId = accountManager.finalizeAccountId(accountName, domainId, projectId, false); } if (isDedicated != null) { SearchCriteria sc1 = createSearchCriteriaForListBgpPeersCmd(id, zoneId, asNumber, keyword); diff --git a/server/src/main/java/org/apache/cloudstack/storage/heuristics/HeuristicRuleHelper.java b/server/src/main/java/org/apache/cloudstack/storage/heuristics/HeuristicRuleHelper.java index 21a34de0d23..2e0780e7fe8 100644 --- a/server/src/main/java/org/apache/cloudstack/storage/heuristics/HeuristicRuleHelper.java +++ b/server/src/main/java/org/apache/cloudstack/storage/heuristics/HeuristicRuleHelper.java @@ -117,8 +117,8 @@ public class HeuristicRuleHelper { accountId = ((SnapshotInfo) obj).getAccountId(); break; case VOLUME: - presetVariables.setVolume(setVolumePresetVariable((VolumeVO) obj)); - accountId = ((VolumeVO) obj).getAccountId(); + presetVariables.setVolume(setVolumePresetVariable((com.cloud.storage.Volume) obj)); + accountId = ((com.cloud.storage.Volume) obj).getAccountId(); break; } presetVariables.setAccount(setAccountPresetVariable(accountId)); @@ -191,14 +191,14 @@ public class HeuristicRuleHelper { return template; } - protected Volume setVolumePresetVariable(VolumeVO volumeVO) { - Volume volume = new Volume(); + protected Volume setVolumePresetVariable(com.cloud.storage.Volume volumeVO) { + Volume volumePresetVariable = new Volume(); - volume.setName(volumeVO.getName()); - volume.setFormat(volumeVO.getFormat()); - volume.setSize(volumeVO.getSize()); + volumePresetVariable.setName(volumeVO.getName()); + volumePresetVariable.setFormat(volumeVO.getFormat()); + volumePresetVariable.setSize(volumeVO.getSize()); - return volume; + return volumePresetVariable; } protected Snapshot setSnapshotPresetVariable(SnapshotInfo snapshotInfo) { diff --git a/server/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateManagerImpl.java b/server/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateManagerImpl.java index ef0f6f6b226..0ebff237a44 100644 --- a/server/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateManagerImpl.java @@ -201,7 +201,14 @@ public class VnfTemplateManagerImpl extends ManagerBase implements VnfTemplateMa } @Override - public void validateVnfApplianceNics(VirtualMachineTemplate template, List networkIds) { + public void validateVnfApplianceNics(VirtualMachineTemplate template, List networkIds, Map vmNetworkMap) { + if (template.isDeployAsIs()) { + if (CollectionUtils.isNotEmpty(networkIds)) { + throw new InvalidParameterValueException("VNF nics mappings should be empty for deploy-as-is templates"); + } + validateVnfApplianceNetworksMap(template, vmNetworkMap); + return; + } if (CollectionUtils.isEmpty(networkIds)) { throw new InvalidParameterValueException("VNF nics list is empty"); } @@ -213,6 +220,18 @@ public class VnfTemplateManagerImpl extends ManagerBase implements VnfTemplateMa } } + private void validateVnfApplianceNetworksMap(VirtualMachineTemplate template, Map vmNetworkMap) { + if (MapUtils.isEmpty(vmNetworkMap)) { + throw new InvalidParameterValueException("VNF networks map is empty"); + } + List vnfNics = vnfTemplateNicDao.listByTemplateId(template.getId()); + for (VnfTemplateNicVO vnfNic : vnfNics) { + if (vnfNic.isRequired() && vmNetworkMap.size() <= vnfNic.getDeviceId()) { + throw new InvalidParameterValueException("VNF nic is required but not found: " + vnfNic); + } + } + } + protected Set getOpenPortsForVnfAppliance(VirtualMachineTemplate template) { Set ports = new HashSet<>(); VnfTemplateDetailVO accessMethodsDetail = vnfTemplateDetailsDao.findDetail(template.getId(), VNF.AccessDetail.ACCESS_METHODS.name().toLowerCase()); diff --git a/server/src/main/java/org/apache/cloudstack/user/UserPasswordResetManager.java b/server/src/main/java/org/apache/cloudstack/user/UserPasswordResetManager.java index 377e57b31e9..ca14f6a1654 100644 --- a/server/src/main/java/org/apache/cloudstack/user/UserPasswordResetManager.java +++ b/server/src/main/java/org/apache/cloudstack/user/UserPasswordResetManager.java @@ -78,7 +78,9 @@ public interface UserPasswordResetManager { ConfigKey UserPasswordResetDomainURL = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, String.class, "user.password.reset.mail.domain.url", null, - "Domain URL for reset password links sent to the user via email", true, + "Domain URL (along with scheme - http:// or https:// and port as applicable) for reset password links sent to the user via email. " + + "If this is not set, CloudStack would determine the domain url based on the first management server from 'host' setting " + + "and http scheme based on the https.enabled flag from server.properties file in the management server.", true, ConfigKey.Scope.Global); void setResetTokenAndSend(UserAccount userAccount); diff --git a/server/src/main/java/org/apache/cloudstack/user/UserPasswordResetManagerImpl.java b/server/src/main/java/org/apache/cloudstack/user/UserPasswordResetManagerImpl.java index 618ad5c8657..c62bca8eca4 100644 --- a/server/src/main/java/org/apache/cloudstack/user/UserPasswordResetManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/user/UserPasswordResetManagerImpl.java @@ -23,6 +23,7 @@ import com.cloud.user.UserVO; import com.cloud.user.dao.UserDao; import com.cloud.utils.StringUtils; import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.server.ServerProperties; import com.github.mustachejava.DefaultMustacheFactory; import com.github.mustachejava.Mustache; import com.github.mustachejava.MustacheFactory; @@ -48,6 +49,7 @@ import java.util.Map; import java.util.Set; import java.util.UUID; +import static org.apache.cloudstack.config.ApiServiceConfiguration.ManagementServerAddresses; import static org.apache.cloudstack.resourcedetail.UserDetailVO.PasswordResetToken; import static org.apache.cloudstack.resourcedetail.UserDetailVO.PasswordResetTokenExpiryDate; @@ -68,7 +70,7 @@ public class UserPasswordResetManagerImpl extends ManagerBase implements UserPas new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, String.class, "user.password.reset.mail.template", "Hello {{username}}!\n" + "You have requested to reset your password. Please click the following link to reset your password:\n" + - "{{{domainUrl}}}{{{resetLink}}}\n" + + "{{{resetLink}}}\n" + "If you did not request a password reset, please ignore this email.\n" + "\n" + "Regards,\n" + @@ -179,10 +181,26 @@ public class UserPasswordResetManagerImpl extends ManagerBase implements UserPas final String email = userAccount.getEmail(); final String username = userAccount.getUsername(); final String subject = "Password Reset Request"; - final String domainUrl = UserPasswordResetDomainURL.value(); + String domainUrl = UserPasswordResetDomainURL.value(); + if (StringUtils.isBlank(domainUrl)) { + String mgmtServerAddr = ManagementServerAddresses.value().split(",")[0]; + if (ServerProperties.isHttpsEnabled()) { + domainUrl = "https://" + mgmtServerAddr + ":" + ServerProperties.getHttpsPort(); + } else { + domainUrl = "http://" + mgmtServerAddr + ":" + ServerProperties.getHttpPort(); + } + } else if (!domainUrl.startsWith("http://") && !domainUrl.startsWith("https://")) { + if (ServerProperties.isHttpsEnabled()) { + domainUrl = "https://" + domainUrl; + } else { + domainUrl = "http://" + domainUrl; + } + } - String resetLink = String.format("/client/#/user/resetPassword?username=%s&token=%s", - username, resetToken); + domainUrl = domainUrl.replaceAll("/+$", ""); + + String resetLink = String.format("%s/client/#/user/resetPassword?username=%s&token=%s", + domainUrl, username, resetToken); String content = getMessageBody(userAccount, resetToken, resetLink); SMTPMailProperties mailProperties = new SMTPMailProperties(); diff --git a/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java b/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java index 13fa2608016..14c67417015 100644 --- a/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java @@ -1550,11 +1550,12 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager { protected VMTemplateVO getTemplateForImportInstance(Long templateId, Hypervisor.HypervisorType hypervisorType) { VMTemplateVO template; if (templateId == null) { - template = templateDao.findByName(VM_IMPORT_DEFAULT_TEMPLATE_NAME); + String templateName = (Hypervisor.HypervisorType.KVM.equals(hypervisorType)) ? KVM_VM_IMPORT_DEFAULT_TEMPLATE_NAME : VM_IMPORT_DEFAULT_TEMPLATE_NAME; + template = templateDao.findByName(templateName); if (template == null) { template = createDefaultDummyVmImportTemplate(Hypervisor.HypervisorType.KVM == hypervisorType); if (template == null) { - throw new InvalidParameterValueException(String.format("Default VM import template with unique name: %s for hypervisor: %s cannot be created. Please use templateid parameter for import", VM_IMPORT_DEFAULT_TEMPLATE_NAME, hypervisorType.toString())); + throw new InvalidParameterValueException(String.format("Default VM import template with unique name: %s for hypervisor: %s cannot be created. Please use templateid parameter for import", templateName, hypervisorType.toString())); } } } else { diff --git a/server/src/main/java/org/apache/cloudstack/vm/schedule/VMSchedulerImpl.java b/server/src/main/java/org/apache/cloudstack/vm/schedule/VMSchedulerImpl.java index 7410fb1c265..56d794fa5c2 100644 --- a/server/src/main/java/org/apache/cloudstack/vm/schedule/VMSchedulerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/vm/schedule/VMSchedulerImpl.java @@ -162,7 +162,13 @@ public class VMSchedulerImpl extends ManagerBase implements VMScheduler, Configu } Date scheduledDateTime = Date.from(ts.toInstant()); - VMScheduledJobVO scheduledJob = new VMScheduledJobVO(vmSchedule.getVmId(), vmSchedule.getId(), vmSchedule.getAction(), scheduledDateTime); + VMScheduledJobVO scheduledJob = vmScheduledJobDao.findByScheduleAndTimestamp(vmSchedule.getId(), scheduledDateTime); + if (scheduledJob != null) { + logger.trace("Job is already scheduled for schedule {} at {}", vmSchedule, scheduledDateTime); + return scheduledDateTime; + } + + scheduledJob = new VMScheduledJobVO(vmSchedule.getVmId(), vmSchedule.getId(), vmSchedule.getAction(), scheduledDateTime); try { vmScheduledJobDao.persist(scheduledJob); ActionEventUtils.onScheduledActionEvent(User.UID_SYSTEM, vm.getAccountId(), actionEventMap.get(vmSchedule.getAction()), diff --git a/server/src/test/java/com/cloud/alert/AlertManagerImplTest.java b/server/src/test/java/com/cloud/alert/AlertManagerImplTest.java index 170fceae986..b5932e8a071 100644 --- a/server/src/test/java/com/cloud/alert/AlertManagerImplTest.java +++ b/server/src/test/java/com/cloud/alert/AlertManagerImplTest.java @@ -16,6 +16,12 @@ // under the License. package com.cloud.alert; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.verify; + import java.io.UnsupportedEncodingException; import java.util.HashMap; import java.util.List; @@ -25,7 +31,10 @@ import javax.mail.MessagingException; import javax.naming.ConfigurationException; import org.apache.cloudstack.backup.BackupManager; +import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.framework.messagebus.MessageBus; +import org.apache.cloudstack.framework.messagebus.MessageSubscriber; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.utils.mailing.SMTPMailSender; @@ -40,6 +49,7 @@ import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; import com.cloud.alert.dao.AlertDao; import com.cloud.capacity.Capacity; @@ -52,16 +62,12 @@ import com.cloud.dc.HostPodVO; import com.cloud.dc.dao.ClusterDao; import com.cloud.dc.dao.DataCenterDao; import com.cloud.dc.dao.HostPodDao; +import com.cloud.event.EventTypes; import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; import com.cloud.storage.StorageManager; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.verify; +import com.cloud.utils.Ternary; @RunWith(MockitoJUnitRunner.class) public class AlertManagerImplTest { @@ -112,6 +118,9 @@ public class AlertManagerImplTest { @Mock ConfigurationDao configDao; + @Mock + MessageBus messageBus; + private final String[] recipients = new String[]{"test@test.com"}; private final String senderAddress = "sender@test.com"; @@ -268,4 +277,81 @@ public class AlertManagerImplTest { assertEquals("Available backup storage space is low, total: 200.0 MB, used: 180.0 MB (90%)", capturedAlert.getContent()); assertEquals(AlertManager.AlertType.ALERT_TYPE_BACKUP_STORAGE.getType(), capturedAlert.getType()); } + + @Test + public void initMessageBusListenerSubscribesToConfigurationEditEvent() { + MessageBus messageBusMock = Mockito.mock(MessageBus.class); + alertManagerImplMock.messageBus = messageBusMock; + alertManagerImplMock.initMessageBusListener(); + Mockito.verify(messageBusMock).subscribe(Mockito.eq(EventTypes.EVENT_CONFIGURATION_VALUE_EDIT), Mockito.any()); + } + + @Test + public void initMessageBusListenerTriggersSetupRepetitiveAlertTypesOnAllowedKeyEdit() { + MessageBus messageBusMock = Mockito.mock(MessageBus.class); + alertManagerImplMock.messageBus = messageBusMock; + alertManagerImplMock.initMessageBusListener(); + ArgumentCaptor captor = ArgumentCaptor.forClass(MessageSubscriber.class); + Mockito.verify(messageBusMock).subscribe(Mockito.eq(EventTypes.EVENT_CONFIGURATION_VALUE_EDIT), captor.capture()); + Ternary args = new Ternary<>(AlertManager.AllowedRepetitiveAlertTypes.key(), ConfigKey.Scope.Global, 1L); + captor.getValue().onPublishMessage(null, null, args); + Mockito.verify(alertManagerImplMock).setupRepetitiveAlertTypes(); + } + + @Test + public void initMessageBusListenerDoesNotTriggerSetupRepetitiveAlertTypesOnOtherKeyEdit() { + MessageBus messageBusMock = Mockito.mock(MessageBus.class); + alertManagerImplMock.messageBus = messageBusMock; + alertManagerImplMock.initMessageBusListener(); + ArgumentCaptor captor = ArgumentCaptor.forClass(MessageSubscriber.class); + Mockito.verify(messageBusMock).subscribe(Mockito.eq(EventTypes.EVENT_CONFIGURATION_VALUE_EDIT), captor.capture()); + Ternary args = new Ternary<>("some.other.key", ConfigKey.Scope.Global, 1L); + captor.getValue().onPublishMessage(null, null, args); + Mockito.verify(alertManagerImplMock, Mockito.never()).setupRepetitiveAlertTypes(); + } + + private void mockAllowedRepetitiveAlertTypesConfigKey(String value) { + ReflectionTestUtils.setField(AlertManager.AllowedRepetitiveAlertTypes, "_defaultValue", value); + } + + @Test + public void setupRepetitiveAlertTypesParsesValidAlertTypesCorrectly() { + mockAllowedRepetitiveAlertTypesConfigKey(AlertManager.AlertType.ALERT_TYPE_CPU.getName() + "," + AlertManager.AlertType.ALERT_TYPE_MEMORY.getName()); + alertManagerImplMock.setupRepetitiveAlertTypes(); + List expectedTypes = (List)ReflectionTestUtils.getField(alertManagerImplMock, "allowedRepetitiveAlertTypeNames"); + Assert.assertNotNull(expectedTypes); + Assert.assertEquals(2, expectedTypes.size()); + Assert.assertTrue(expectedTypes.contains(AlertManager.AlertType.ALERT_TYPE_CPU.getName().toLowerCase())); + Assert.assertTrue(expectedTypes.contains(AlertManager.AlertType.ALERT_TYPE_MEMORY.getName().toLowerCase())); + } + + @Test + public void setupRepetitiveAlertTypesHandlesEmptyConfigValue() { + mockAllowedRepetitiveAlertTypesConfigKey(""); + alertManagerImplMock.setupRepetitiveAlertTypes(); + List expectedTypes = (List)ReflectionTestUtils.getField(alertManagerImplMock, "allowedRepetitiveAlertTypeNames"); + Assert.assertNotNull(expectedTypes); + Assert.assertTrue(expectedTypes.isEmpty()); + } + + @Test + public void setupRepetitiveAlertTypesIgnoresCustomAlertTypes() { + String customAlertTypeName = "CUSTOM_ALERT_TYPE"; + mockAllowedRepetitiveAlertTypesConfigKey(AlertManager.AlertType.ALERT_TYPE_CPU.getName() + "," + customAlertTypeName); + alertManagerImplMock.setupRepetitiveAlertTypes(); + List expectedTypes = (List)ReflectionTestUtils.getField(alertManagerImplMock, "allowedRepetitiveAlertTypeNames"); + Assert.assertNotNull(expectedTypes); + Assert.assertEquals(2, expectedTypes.size()); + Assert.assertTrue(expectedTypes.contains(AlertManager.AlertType.ALERT_TYPE_CPU.getName().toLowerCase())); + Assert.assertTrue(expectedTypes.contains(customAlertTypeName.toLowerCase())); + } + + @Test + public void setupRepetitiveAlertTypesHandlesNullConfigValue() { + mockAllowedRepetitiveAlertTypesConfigKey(null); + alertManagerImplMock.setupRepetitiveAlertTypes(); + List expectedTypes = (List)ReflectionTestUtils.getField(alertManagerImplMock, "allowedRepetitiveAlertTypeNames"); + Assert.assertNotNull(expectedTypes); + Assert.assertTrue(expectedTypes.isEmpty()); + } } diff --git a/server/src/test/java/com/cloud/api/ApiServletTest.java b/server/src/test/java/com/cloud/api/ApiServletTest.java index 4d4f0a12098..c5ee9f58154 100644 --- a/server/src/test/java/com/cloud/api/ApiServletTest.java +++ b/server/src/test/java/com/cloud/api/ApiServletTest.java @@ -16,35 +16,8 @@ // under the License. package com.cloud.api; -import com.cloud.api.auth.ListUserTwoFactorAuthenticatorProvidersCmd; -import com.cloud.api.auth.SetupUserTwoFactorAuthenticationCmd; -import com.cloud.api.auth.ValidateUserTwoFactorAuthenticationCodeCmd; -import com.cloud.server.ManagementServer; -import com.cloud.user.Account; -import com.cloud.user.AccountManagerImpl; -import com.cloud.user.AccountService; -import com.cloud.user.User; -import com.cloud.user.UserAccount; -import com.cloud.utils.HttpUtils; -import com.cloud.vm.UserVmManager; -import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.auth.APIAuthenticationManager; -import org.apache.cloudstack.api.auth.APIAuthenticationType; -import org.apache.cloudstack.api.auth.APIAuthenticator; -import org.apache.cloudstack.api.command.admin.config.ListCfgsByCmd; -import org.apache.cloudstack.framework.config.ConfigKey; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.junit.MockitoJUnitRunner; +import static org.mockito.ArgumentMatchers.nullable; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import javax.servlet.http.HttpSession; import java.io.IOException; import java.io.PrintWriter; import java.io.StringWriter; @@ -56,11 +29,46 @@ import java.net.UnknownHostException; import java.util.HashMap; import java.util.Map; -import static org.mockito.ArgumentMatchers.nullable; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.servlet.http.HttpSession; + +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.auth.APIAuthenticationManager; +import org.apache.cloudstack.api.auth.APIAuthenticationType; +import org.apache.cloudstack.api.auth.APIAuthenticator; +import org.apache.cloudstack.api.command.admin.config.ListCfgsByCmd; +import org.apache.cloudstack.api.command.admin.offering.IsAccountAllowedToCreateOfferingsWithTagsCmd; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.impl.ConfigDepotImpl; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.api.auth.ListUserTwoFactorAuthenticatorProvidersCmd; +import com.cloud.api.auth.SetupUserTwoFactorAuthenticationCmd; +import com.cloud.api.auth.ValidateUserTwoFactorAuthenticationCodeCmd; +import com.cloud.server.ManagementServer; +import com.cloud.user.Account; +import com.cloud.user.AccountManagerImpl; +import com.cloud.user.AccountService; +import com.cloud.user.User; +import com.cloud.user.UserAccount; +import com.cloud.utils.HttpUtils; +import com.cloud.vm.UserVmManager; @RunWith(MockitoJUnitRunner.class) public class ApiServletTest { + private static final String[] STATE_CHANGING_COMMAND_CHECK_NAME_PARAM = + {ApiServer.EnforcePostRequestsAndTimestamps.key()}; + @Mock ApiServer apiServer; @@ -97,17 +105,20 @@ public class ApiServletTest { @Mock AccountService accountMgr; - @Mock ConfigKey useForwardHeader; + @Mock + ConfigDepotImpl mockConfigDepot; + StringWriter responseWriter; ApiServlet servlet; - ApiServlet spyServlet; + + private ConfigDepotImpl originalConfigDepot; + @SuppressWarnings("unchecked") @Before public void setup() throws SecurityException, NoSuchFieldException, - IllegalArgumentException, IllegalAccessException, IOException, UnknownHostException { + IllegalArgumentException, IllegalAccessException, IOException { servlet = new ApiServlet(); - spyServlet = Mockito.spy(servlet); responseWriter = new StringWriter(); Mockito.when(response.getWriter()).thenReturn( new PrintWriter(responseWriter)); @@ -131,6 +142,7 @@ public class ApiServletTest { apiServerField.setAccessible(true); apiServerField.set(servlet, apiServer); + setupConfigDepotMock(); } /** @@ -151,6 +163,33 @@ public class ApiServletTest { Field smsField = ApiDBUtils.class.getDeclaredField("s_ms"); smsField.setAccessible(true); smsField.set(null, null); + restoreConfigDepot(); + } + + private void setupConfigDepotMock() throws NoSuchFieldException, IllegalAccessException { + Field depotField = ConfigKey.class.getDeclaredField("s_depot"); + depotField.setAccessible(true); + originalConfigDepot = (ConfigDepotImpl) depotField.get(null); + depotField.set(null, mockConfigDepot); + Mockito.when(mockConfigDepot.getConfigStringValue( + Mockito.anyString(), + Mockito.any(ConfigKey.Scope.class), + Mockito.any() + )).thenReturn(null); + } + + private void restoreConfigDepot() throws Exception { + Field depotField = ConfigKey.class.getDeclaredField("s_depot"); + depotField.setAccessible(true); + depotField.set(null, originalConfigDepot); + } + + private void setConfigValue(String configName, String value) { + Mockito.when(mockConfigDepot.getConfigStringValue( + Mockito.eq(configName), + Mockito.eq(ConfigKey.Scope.Global), + Mockito.isNull() + )).thenReturn(value); } @Test @@ -261,43 +300,40 @@ public class ApiServletTest { @Test public void getClientAddressWithXForwardedFor() throws UnknownHostException { - String[] proxynet = {"127.0.0.0/8"}; - Mockito.when(spyServlet.proxyNets()).thenReturn(proxynet); - Mockito.when(spyServlet.doUseForwardHeaders()).thenReturn(true); + setConfigValue("proxy.header.verify", "true"); + setConfigValue("proxy.cidr", "127.0.0.0/8"); + Mockito.when(request.getRemoteAddr()).thenReturn("127.0.0.1"); Mockito.when(request.getHeader(Mockito.eq("X-Forwarded-For"))).thenReturn("192.168.1.1"); - Assert.assertEquals(InetAddress.getByName("192.168.1.1"), spyServlet.getClientAddress(request)); + Assert.assertEquals(InetAddress.getByName("192.168.1.1"), ApiServlet.getClientAddress(request)); } @Test public void getClientAddressWithHttpXForwardedFor() throws UnknownHostException { - String[] proxynet = {"127.0.0.0/8"}; - Mockito.when(spyServlet.proxyNets()).thenReturn(proxynet); - Mockito.when(spyServlet.doUseForwardHeaders()).thenReturn(true); + setConfigValue("proxy.header.verify", "true"); + setConfigValue("proxy.cidr", "127.0.0.0/8"); Mockito.when(request.getHeader(Mockito.eq("HTTP_X_FORWARDED_FOR"))).thenReturn("192.168.1.1"); - Assert.assertEquals(InetAddress.getByName("192.168.1.1"), spyServlet.getClientAddress(request)); + Assert.assertEquals(InetAddress.getByName("192.168.1.1"), ApiServlet.getClientAddress(request)); } @Test public void getClientAddressWithRemoteAddr() throws UnknownHostException { - String[] proxynet = {"127.0.0.0/8"}; - Mockito.when(spyServlet.proxyNets()).thenReturn(proxynet); - Mockito.when(spyServlet.doUseForwardHeaders()).thenReturn(true); - Assert.assertEquals(InetAddress.getByName("127.0.0.1"), spyServlet.getClientAddress(request)); + setConfigValue("proxy.header.verify", "true"); + setConfigValue("proxy.cidr", "127.0.0.0/8"); + Assert.assertEquals(InetAddress.getByName("127.0.0.1"), ApiServlet.getClientAddress(request)); } @Test public void getClientAddressWithHttpClientIp() throws UnknownHostException { - String[] proxynet = {"127.0.0.0/8"}; - Mockito.when(spyServlet.proxyNets()).thenReturn(proxynet); - Mockito.when(spyServlet.doUseForwardHeaders()).thenReturn(true); + setConfigValue("proxy.header.verify", "true"); + setConfigValue("proxy.cidr", "127.0.0.0/8"); Mockito.when(request.getHeader(Mockito.eq("HTTP_CLIENT_IP"))).thenReturn("192.168.1.1"); - Assert.assertEquals(InetAddress.getByName("192.168.1.1"), spyServlet.getClientAddress(request)); + Assert.assertEquals(InetAddress.getByName("192.168.1.1"), ApiServlet.getClientAddress(request)); } @Test public void getClientAddressDefault() throws UnknownHostException { Mockito.when(request.getRemoteAddr()).thenReturn("127.0.0.1"); - Assert.assertEquals(InetAddress.getByName("127.0.0.1"), spyServlet.getClientAddress(request)); + Assert.assertEquals(InetAddress.getByName("127.0.0.1"), ApiServlet.getClientAddress(request)); } @Test @@ -432,4 +468,88 @@ public class ApiServletTest { Assert.assertEquals(false, result); } + + @Test + public void isStateChangingCommandNotUsingPOSTReturnsFalseForPostMethod() { + String command = "updateConfiguration"; + String method = "POST"; + Map params = new HashMap<>(); + + boolean result = servlet.isStateChangingCommandNotUsingPOST(command, method, params); + + Assert.assertFalse(result); + } + + @Test + public void isStateChangingCommandNotUsingPOSTReturnsTrueForNullCommandAndMethod() { + String command = null; + String method = null; + Map params = new HashMap<>(); + + boolean result = servlet.isStateChangingCommandNotUsingPOST(command, method, params); + + Assert.assertTrue(result); + } + + @Test + public void isStateChangingCommandNotUsingPOSTReturnsFalseForGetHttpMethodAnnotation() { + String command = "isAccountAllowedToCreateOfferingsWithTags"; + String method = "GET"; + Map params = new HashMap<>(); + Class cmdClass = IsAccountAllowedToCreateOfferingsWithTagsCmd.class; + APICommand apiCommand = cmdClass.getAnnotation(APICommand.class); + Mockito.doReturn(cmdClass).when(apiServer).getCmdClass(command); + Assert.assertNotNull(apiCommand); + Assert.assertEquals("GET", apiCommand.httpMethod()); + boolean result = servlet.isStateChangingCommandNotUsingPOST(command, method, params); + Assert.assertFalse(result); + } + + @Test + public void isStateChangingCommandNotUsingPOSTReturnsFalseForMatchingGetRequestPattern() { + String command = "listZones"; + String method = "GET"; + Map params = new HashMap<>(); + boolean result = servlet.isStateChangingCommandNotUsingPOST(command, method, params); + Assert.assertFalse(result); + } + + @Test + public void isStateChangingCommandNotUsingPOSTReturnsTrueForMissingNameParameter() { + String command = "updateConfiguration"; + String method = "GET"; + Map params = new HashMap<>(); + boolean result = servlet.isStateChangingCommandNotUsingPOST(command, method, params); + Assert.assertTrue(result); + } + + @Test + public void isStateChangingCommandNotUsingPOSTReturnsFalseForUpdateConfigurationEnforcePostRequestsKey() { + String command = "updateConfiguration"; + String method = "GET"; + Map params = new HashMap<>(); + params.put("name", STATE_CHANGING_COMMAND_CHECK_NAME_PARAM); + boolean result = servlet.isStateChangingCommandNotUsingPOST(command, method, params); + Assert.assertFalse(result); + } + + @Test + public void isStateChangingCommandNotUsingPOSTReturnsFalseForWrongApiEnforcePostRequestsKey() { + String command = "updateSomeApi"; + String method = "GET"; + Map params = new HashMap<>(); + params.put("name", STATE_CHANGING_COMMAND_CHECK_NAME_PARAM); + boolean result = servlet.isStateChangingCommandNotUsingPOST(command, method, params); + Assert.assertTrue(result); + } + + @Test + public void isStateChangingCommandNotUsingPOSTReturnsFalseForUpdateConfigurationNonEnforcePostRequestsKey() { + String command = "updateConfiguration"; + String method = "GET"; + Map params = new HashMap<>(); + params.put("name", new String[] { "key" }); + boolean result = servlet.isStateChangingCommandNotUsingPOST(command, method, params); + Assert.assertTrue(result); + } } diff --git a/server/src/test/java/com/cloud/resource/ResourceManagerImplTest.java b/server/src/test/java/com/cloud/resource/ResourceManagerImplTest.java index 15d23775742..47472408404 100644 --- a/server/src/test/java/com/cloud/resource/ResourceManagerImplTest.java +++ b/server/src/test/java/com/cloud/resource/ResourceManagerImplTest.java @@ -57,6 +57,7 @@ import org.apache.cloudstack.api.command.admin.host.CancelHostAsDegradedCmd; import org.apache.cloudstack.api.command.admin.host.DeclareHostAsDegradedCmd; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreInfo; +import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; @@ -75,6 +76,7 @@ import org.mockito.MockitoAnnotations; import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; +import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -179,6 +181,12 @@ public class ResourceManagerImplTest { private MockedConstruction getVncPortCommandMockedConstruction; private AutoCloseable closeable; + private void overrideDefaultConfigValue(final ConfigKey configKey, final String name, final Object o) throws IllegalAccessException, NoSuchFieldException { + Field f = ConfigKey.class.getDeclaredField(name); + f.setAccessible(true); + f.set(configKey, o); + } + @Before public void setup() throws Exception { closeable = MockitoAnnotations.openMocks(this); @@ -221,12 +229,12 @@ public class ResourceManagerImplTest { eq("service cloudstack-agent restart"))). willReturn(new SSHCmdHelper.SSHCmdResult(0,"","")); - when(configurationDao.getValue(ResourceManager.KvmSshToAgentEnabled.key())).thenReturn("true"); + overrideDefaultConfigValue(ResourceManager.KvmSshToAgentEnabled, "_defaultValue", "true"); rootDisks = Arrays.asList(rootDisk1, rootDisk2); dataDisks = Collections.singletonList(dataDisk); - when(volumeDao.findByPoolId(poolId)).thenReturn(rootDisks); - when(volumeDao.findByPoolId(poolId, Volume.Type.DATADISK)).thenReturn(dataDisks); + when(volumeDao.findNonDestroyedVolumesByPoolId(poolId)).thenReturn(rootDisks); + when(volumeDao.findNonDestroyedVolumesByPoolId(poolId, Volume.Type.DATADISK)).thenReturn(dataDisks); } @After @@ -399,9 +407,9 @@ public class ResourceManagerImplTest { } @Test(expected = CloudRuntimeException.class) - public void testHandleAgentSSHDisabledNotConnectedAgent() { + public void testHandleAgentSSHDisabledNotConnectedAgent() throws NoSuchFieldException, IllegalAccessException { when(host.getStatus()).thenReturn(Status.Disconnected); - when(configurationDao.getValue(ResourceManager.KvmSshToAgentEnabled.key())).thenReturn("false"); + overrideDefaultConfigValue(ResourceManager.KvmSshToAgentEnabled, "_defaultValue", "false"); resourceManager.handleAgentIfNotConnected(host, false); } @@ -591,22 +599,22 @@ public class ResourceManagerImplTest { @Test public void testDestroyLocalStoragePoolVolumesOnlyRootDisks() { - when(volumeDao.findByPoolId(poolId, Volume.Type.DATADISK)).thenReturn(null); + when(volumeDao.findNonDestroyedVolumesByPoolId(poolId, Volume.Type.DATADISK)).thenReturn(null); resourceManager.destroyLocalStoragePoolVolumes(poolId); verify(volumeDao, times(rootDisks.size())).updateAndRemoveVolume(any(VolumeVO.class)); } @Test public void testDestroyLocalStoragePoolVolumesOnlyDataDisks() { - when(volumeDao.findByPoolId(poolId)).thenReturn(null); + when(volumeDao.findNonDestroyedVolumesByPoolId(poolId)).thenReturn(null); resourceManager.destroyLocalStoragePoolVolumes(poolId); verify(volumeDao, times(dataDisks.size())).updateAndRemoveVolume(any(VolumeVO.class)); } @Test public void testDestroyLocalStoragePoolVolumesNoDisks() { - when(volumeDao.findByPoolId(poolId)).thenReturn(null); - when(volumeDao.findByPoolId(poolId, Volume.Type.DATADISK)).thenReturn(null); + when(volumeDao.findNonDestroyedVolumesByPoolId(poolId)).thenReturn(null); + when(volumeDao.findNonDestroyedVolumesByPoolId(poolId, Volume.Type.DATADISK)).thenReturn(null); resourceManager.destroyLocalStoragePoolVolumes(poolId); verify(volumeDao, never()).updateAndRemoveVolume(any(VolumeVO.class)); } @@ -944,7 +952,7 @@ public class ResourceManagerImplTest { Mockito.when(volume2.getInstanceId()).thenReturn(101L); List volumesInPool = Arrays.asList(volume1, volume2); - Mockito.doReturn(volumesInPool).when(volumeDao).findByPoolId(poolId); + Mockito.doReturn(volumesInPool).when(volumeDao).findNonDestroyedVolumesByPoolId(poolId); VMInstanceVO vmInstance1 = Mockito.mock(VMInstanceVO.class); VMInstanceVO vmInstance2 = Mockito.mock(VMInstanceVO.class); diff --git a/server/src/test/java/com/cloud/resourcelimit/ResourceLimitManagerImplTest.java b/server/src/test/java/com/cloud/resourcelimit/ResourceLimitManagerImplTest.java index 0b0b8c5e43f..84586bcdc2c 100644 --- a/server/src/test/java/com/cloud/resourcelimit/ResourceLimitManagerImplTest.java +++ b/server/src/test/java/com/cloud/resourcelimit/ResourceLimitManagerImplTest.java @@ -16,17 +16,17 @@ // under the License. package com.cloud.resourcelimit; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Arrays; +import java.util.EnumSet; import java.util.HashMap; import java.util.List; import java.util.Map; -import com.cloud.event.ActionEventUtils; -import com.cloud.event.EventTypes; -import com.cloud.utils.db.EntityManager; - import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.response.AccountResponse; import org.apache.cloudstack.api.response.DomainResponse; @@ -34,6 +34,7 @@ import org.apache.cloudstack.api.response.TaggedResourceLimitAndCountResponse; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.reservation.dao.ReservationDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; @@ -61,6 +62,8 @@ import com.cloud.configuration.dao.ResourceLimitDao; import com.cloud.domain.Domain; import com.cloud.domain.DomainVO; import com.cloud.domain.dao.DomainDao; +import com.cloud.event.ActionEventUtils; +import com.cloud.event.EventTypes; import com.cloud.exception.ResourceAllocationException; import com.cloud.offering.DiskOffering; import com.cloud.offering.ServiceOffering; @@ -74,21 +77,19 @@ import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.template.VirtualMachineTemplate; import com.cloud.user.Account; -import com.cloud.user.User; import com.cloud.user.AccountManager; import com.cloud.user.AccountVO; import com.cloud.user.ResourceLimitService; +import com.cloud.user.User; import com.cloud.user.dao.AccountDao; import com.cloud.utils.Pair; +import com.cloud.utils.db.EntityManager; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineManager; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.VMInstanceDao; import com.cloud.vpc.MockResourceLimitManagerImpl; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - @RunWith(MockitoJUnitRunner.class) public class ResourceLimitManagerImplTest { private Logger logger = LogManager.getLogger(ResourceLimitManagerImplTest.class); @@ -129,6 +130,8 @@ public class ResourceLimitManagerImplTest { UserVmDao userVmDao; @Mock EntityManager entityManager; + @Mock + SnapshotDataStoreDao snapshotDataStoreDao; private CallContext callContext; private List hostTags = List.of("htag1", "htag2", "htag3"); @@ -900,12 +903,13 @@ public class ResourceLimitManagerImplTest { String tag = null; Mockito.when(vmDao.findIdsOfAllocatedVirtualRoutersForAccount(accountId)) .thenReturn(List.of(1L)); + Mockito.when(snapshotDataStoreDao.getSnapshotsPhysicalSizeOnPrimaryStorageByAccountId(accountId)).thenReturn(100L); Mockito.when(volumeDao.primaryStorageUsedForAccount(Mockito.eq(accountId), Mockito.anyList())).thenReturn(100L); - Assert.assertEquals(100L, resourceLimitManager.calculatePrimaryStorageForAccount(accountId, tag)); + Assert.assertEquals(200L, resourceLimitManager.calculatePrimaryStorageForAccount(accountId, tag)); tag = ""; Mockito.when(volumeDao.primaryStorageUsedForAccount(Mockito.eq(accountId), Mockito.anyList())).thenReturn(200L); - Assert.assertEquals(200L, resourceLimitManager.calculatePrimaryStorageForAccount(accountId, tag)); + Assert.assertEquals(300L, resourceLimitManager.calculatePrimaryStorageForAccount(accountId, tag)); tag = "tag"; VolumeVO vol = Mockito.mock(VolumeVO.class); @@ -913,7 +917,7 @@ public class ResourceLimitManagerImplTest { Mockito.when(vol.getSize()).thenReturn(size); List vols = List.of(vol, vol); Mockito.doReturn(vols).when(resourceLimitManager).getVolumesWithAccountAndTag(accountId, tag); - Assert.assertEquals(vols.size() * size, resourceLimitManager.calculatePrimaryStorageForAccount(accountId, tag)); + Assert.assertEquals((vols.size() * size) + 100L, resourceLimitManager.calculatePrimaryStorageForAccount(accountId, tag)); } @Test @@ -1394,4 +1398,53 @@ public class ResourceLimitManagerImplTest { domainId, ApiCommandResourceType.Domain.toString())); } } + + @Test + public void consolidatedResourceLimitsForAllResourceTypesWithAccountId() { + Long accountId = 1L; + Long domainId = null; + List foundLimits = new ArrayList<>(); + ResourceLimitVO limit = new ResourceLimitVO(Resource.ResourceType.cpu, 10L, accountId, Resource.ResourceOwnerType.Account); + foundLimits.add(limit); + + Mockito.when(accountManager.getAccount(accountId)).thenReturn(Mockito.mock(Account.class)); + Mockito.doReturn(20L).when(resourceLimitManager).findCorrectResourceLimitForAccount(Mockito.any(Account.class), Mockito.any(Resource.ResourceType.class), Mockito.isNull()); + + List result = resourceLimitManager.getConsolidatedResourceLimitsForAllResourceTypes(accountId, domainId, foundLimits, true); + + Assert.assertEquals(EnumSet.allOf(Resource.ResourceType.class).size(), result.size()); + Assert.assertTrue(result.contains(limit)); + } + + @Test + public void consolidatedResourceLimitsForAllResourceTypesWithDomainId() { + Long accountId = null; + Long domainId = 1L; + List foundLimits = new ArrayList<>(); + ResourceLimitVO limit = new ResourceLimitVO(Resource.ResourceType.memory, 15L, domainId, Resource.ResourceOwnerType.Domain); + foundLimits.add(limit); + + Mockito.when(domainDao.findById(domainId)).thenReturn(Mockito.mock(DomainVO.class)); + Mockito.doReturn(30L).when(resourceLimitManager).findCorrectResourceLimitForDomain(Mockito.any(Domain.class), Mockito.any(Resource.ResourceType.class), Mockito.isNull()); + + List result = resourceLimitManager.getConsolidatedResourceLimitsForAllResourceTypes(accountId, domainId, foundLimits, false); + + Assert.assertEquals(EnumSet.allOf(Resource.ResourceType.class).size(), result.size()); + Assert.assertTrue(result.contains(limit)); + } + + @Test + public void consolidatedResourceLimitsForAllResourceTypesWithEmptyFoundLimits() { + Long accountId = 1L; + Long domainId = null; + List foundLimits = new ArrayList<>(); + + Mockito.when(accountManager.getAccount(accountId)).thenReturn(Mockito.mock(Account.class)); + Mockito.doReturn(25L).when(resourceLimitManager).findCorrectResourceLimitForAccount(Mockito.any(Account.class), Mockito.any(Resource.ResourceType.class), Mockito.isNull()); + + List result = resourceLimitManager.getConsolidatedResourceLimitsForAllResourceTypes(accountId, domainId, foundLimits, true); + + Assert.assertEquals(EnumSet.allOf(Resource.ResourceType.class).size(), result.size()); + Assert.assertEquals(25L, result.get(0).getMax().longValue()); + } } diff --git a/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java b/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java index 576c32c08ba..8f88800d549 100644 --- a/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java +++ b/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java @@ -575,7 +575,7 @@ public class StorageManagerImplTest { } @Test - public void getStoragePoolNonDestroyedVolumesLogTestNonDestroyedVolumesReturnLog() { + public void getStoragePoolNonDestroyedVolumesLogTestNonDestroyedVolumes_VMAttachedLogs() { Mockito.doReturn(1L).when(storagePoolVOMock).getId(); Mockito.doReturn(1L).when(volume1VOMock).getInstanceId(); Mockito.doReturn("786633d1-a942-4374-9d56-322dd4b0d202").when(volume1VOMock).getUuid(); @@ -583,7 +583,7 @@ public class StorageManagerImplTest { Mockito.doReturn("ffb46333-e983-4c21-b5f0-51c5877a3805").when(volume2VOMock).getUuid(); Mockito.doReturn("58760044-928f-4c4e-9fef-d0e48423595e").when(vmInstanceVOMock).getUuid(); - Mockito.when(_volumeDao.findByPoolId(storagePoolVOMock.getId(), null)).thenReturn(List.of(volume1VOMock, volume2VOMock)); + Mockito.when(_volumeDao.findNonDestroyedVolumesByPoolId(storagePoolVOMock.getId(), null)).thenReturn(List.of(volume1VOMock, volume2VOMock)); Mockito.doReturn(vmInstanceVOMock).when(vmInstanceDao).findById(Mockito.anyLong()); String log = storageManagerImpl.getStoragePoolNonDestroyedVolumesLog(storagePoolVOMock.getId()); @@ -592,6 +592,58 @@ public class StorageManagerImplTest { Assert.assertEquals(expected, log); } + @Test + public void getStoragePoolNonDestroyedVolumesLogTestNonDestroyedVolumes_VMLogForOneVolume() { + Mockito.doReturn(1L).when(storagePoolVOMock).getId(); + Mockito.doReturn(null).when(volume1VOMock).getInstanceId(); + Mockito.doReturn("786633d1-a942-4374-9d56-322dd4b0d202").when(volume1VOMock).getUuid(); + Mockito.doReturn(1L).when(volume2VOMock).getInstanceId(); + Mockito.doReturn("ffb46333-e983-4c21-b5f0-51c5877a3805").when(volume2VOMock).getUuid(); + Mockito.doReturn("58760044-928f-4c4e-9fef-d0e48423595e").when(vmInstanceVOMock).getUuid(); + + Mockito.when(_volumeDao.findNonDestroyedVolumesByPoolId(storagePoolVOMock.getId(), null)).thenReturn(List.of(volume1VOMock, volume2VOMock)); + Mockito.doReturn(vmInstanceVOMock).when(vmInstanceDao).findById(Mockito.anyLong()); + + String log = storageManagerImpl.getStoragePoolNonDestroyedVolumesLog(storagePoolVOMock.getId()); + String expected = String.format("[Volume [%s] (not attached to any VM), Volume [%s] (attached to VM [%s])]", volume1VOMock.getUuid(), volume2VOMock.getUuid(), vmInstanceVOMock.getUuid()); + + Assert.assertEquals(expected, log); + } + + @Test + public void getStoragePoolNonDestroyedVolumesLogTestNonDestroyedVolumes_NotAttachedLogs() { + Mockito.doReturn(1L).when(storagePoolVOMock).getId(); + Mockito.doReturn(null).when(volume1VOMock).getInstanceId(); + Mockito.doReturn("786633d1-a942-4374-9d56-322dd4b0d202").when(volume1VOMock).getUuid(); + Mockito.doReturn(null).when(volume2VOMock).getInstanceId(); + Mockito.doReturn("ffb46333-e983-4c21-b5f0-51c5877a3805").when(volume2VOMock).getUuid(); + + Mockito.when(_volumeDao.findNonDestroyedVolumesByPoolId(storagePoolVOMock.getId(), null)).thenReturn(List.of(volume1VOMock, volume2VOMock)); + + String log = storageManagerImpl.getStoragePoolNonDestroyedVolumesLog(storagePoolVOMock.getId()); + String expected = String.format("[Volume [%s] (not attached to any VM), Volume [%s] (not attached to any VM)]", volume1VOMock.getUuid(), volume2VOMock.getUuid()); + + Assert.assertEquals(expected, log); + } + + @Test + public void getStoragePoolNonDestroyedVolumesLogTestNonDestroyedVolumes_VMNotExistsLog() { + Mockito.doReturn(1L).when(storagePoolVOMock).getId(); + Mockito.doReturn(1L).when(volume1VOMock).getInstanceId(); + Mockito.doReturn("786633d1-a942-4374-9d56-322dd4b0d202").when(volume1VOMock).getUuid(); + Mockito.doReturn(1L).when(volume2VOMock).getInstanceId(); + Mockito.doReturn("ffb46333-e983-4c21-b5f0-51c5877a3805").when(volume2VOMock).getUuid(); + + Mockito.when(_volumeDao.findNonDestroyedVolumesByPoolId(storagePoolVOMock.getId(), null)).thenReturn(List.of(volume1VOMock, volume2VOMock)); + Mockito.doReturn(null).when(vmInstanceDao).findById(Mockito.anyLong()); + + String log = storageManagerImpl.getStoragePoolNonDestroyedVolumesLog(storagePoolVOMock.getId()); + String expected = String.format("[Volume [%s] (attached VM with ID [%d] doesn't exists), Volume [%s] (attached VM with ID [%d] doesn't exists)]", + volume1VOMock.getUuid(), volume1VOMock.getInstanceId(), volume2VOMock.getUuid(), volume2VOMock.getInstanceId()); + + Assert.assertEquals(expected, log); + } + private ChangeStoragePoolScopeCmd mockChangeStoragePooolScopeCmd(String newScope) { ChangeStoragePoolScopeCmd cmd = new ChangeStoragePoolScopeCmd(); ReflectionTestUtils.setField(cmd, "id", 1L); diff --git a/server/src/test/java/com/cloud/storage/snapshot/SnapshotManagerImplTest.java b/server/src/test/java/com/cloud/storage/snapshot/SnapshotManagerImplTest.java index 367a49a801f..2d3cb04ab96 100644 --- a/server/src/test/java/com/cloud/storage/snapshot/SnapshotManagerImplTest.java +++ b/server/src/test/java/com/cloud/storage/snapshot/SnapshotManagerImplTest.java @@ -30,6 +30,7 @@ import com.cloud.storage.Snapshot; import com.cloud.storage.SnapshotPolicyVO; import com.cloud.storage.SnapshotVO; import com.cloud.storage.VolumeVO; +import com.cloud.server.TaggedResourceService; import com.cloud.storage.dao.SnapshotDao; import com.cloud.storage.dao.SnapshotPolicyDao; import com.cloud.storage.dao.SnapshotZoneDao; @@ -44,6 +45,7 @@ import com.cloud.utils.Pair; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; +import org.apache.cloudstack.api.command.user.snapshot.DeleteSnapshotPoliciesCmd; import org.apache.cloudstack.api.command.user.snapshot.ListSnapshotPoliciesCmd; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult; @@ -100,6 +102,10 @@ public class SnapshotManagerImplTest { VolumeDao volumeDao; @Mock SnapshotPolicyDao snapshotPolicyDao; + @Mock + SnapshotScheduler snapshotScheduler; + @Mock + TaggedResourceService taggedResourceService; @InjectMocks SnapshotManagerImpl snapshotManager = new SnapshotManagerImpl(); @@ -108,6 +114,8 @@ public class SnapshotManagerImplTest { snapshotManager._snapshotPolicyDao = snapshotPolicyDao; snapshotManager._volsDao = volumeDao; snapshotManager._accountMgr = accountManager; + snapshotManager._snapSchedMgr = snapshotScheduler; + snapshotManager.taggedResourceService = taggedResourceService; } @After @@ -520,4 +528,88 @@ public class SnapshotManagerImplTest { Assert.assertEquals(1, result.first().size()); Assert.assertEquals(Integer.valueOf(1), result.second()); } + + @Test + public void testDeleteSnapshotPoliciesForRemovedVolume() { + Long policyId = 1L; + Long volumeId = 10L; + Long accountId = 2L; + + DeleteSnapshotPoliciesCmd cmd = Mockito.mock(DeleteSnapshotPoliciesCmd.class); + Mockito.when(cmd.getId()).thenReturn(policyId); + Mockito.when(cmd.getIds()).thenReturn(null); + + Account caller = Mockito.mock(Account.class); + Mockito.when(caller.getId()).thenReturn(accountId); + CallContext.register(Mockito.mock(User.class), caller); + + SnapshotPolicyVO policyVO = Mockito.mock(SnapshotPolicyVO.class); + Mockito.when(policyVO.getId()).thenReturn(policyId); + Mockito.when(policyVO.getVolumeId()).thenReturn(volumeId); + Mockito.when(policyVO.getUuid()).thenReturn("policy-uuid"); + Mockito.when(snapshotPolicyDao.findById(policyId)).thenReturn(policyVO); + + // Volume is removed (expunged) but findByIdIncludingRemoved should still return it + VolumeVO volumeVO = Mockito.mock(VolumeVO.class); + Mockito.when(volumeDao.findByIdIncludingRemoved(volumeId)).thenReturn(volumeVO); + + Mockito.when(snapshotPolicyDao.remove(policyId)).thenReturn(true); + + boolean result = snapshotManager.deleteSnapshotPolicies(cmd); + + Assert.assertTrue(result); + Mockito.verify(volumeDao).findByIdIncludingRemoved(volumeId); + Mockito.verify(snapshotScheduler).removeSchedule(volumeId, policyId); + Mockito.verify(snapshotPolicyDao).remove(policyId); + } + + @Test(expected = InvalidParameterValueException.class) + public void testDeleteSnapshotPoliciesNoPolicyId() { + DeleteSnapshotPoliciesCmd cmd = Mockito.mock(DeleteSnapshotPoliciesCmd.class); + Mockito.when(cmd.getId()).thenReturn(null); + Mockito.when(cmd.getIds()).thenReturn(null); + + snapshotManager.deleteSnapshotPolicies(cmd); + } + + @Test(expected = InvalidParameterValueException.class) + public void testDeleteSnapshotPoliciesPolicyNotFound() { + Long policyId = 1L; + + DeleteSnapshotPoliciesCmd cmd = Mockito.mock(DeleteSnapshotPoliciesCmd.class); + Mockito.when(cmd.getId()).thenReturn(policyId); + Mockito.when(cmd.getIds()).thenReturn(null); + + Mockito.when(snapshotPolicyDao.findById(policyId)).thenReturn(null); + + snapshotManager.deleteSnapshotPolicies(cmd); + } + + @Test(expected = InvalidParameterValueException.class) + public void testDeleteSnapshotPoliciesVolumeNotFound() { + Long policyId = 1L; + Long volumeId = 10L; + + DeleteSnapshotPoliciesCmd cmd = Mockito.mock(DeleteSnapshotPoliciesCmd.class); + Mockito.when(cmd.getId()).thenReturn(policyId); + Mockito.when(cmd.getIds()).thenReturn(null); + + SnapshotPolicyVO policyVO = Mockito.mock(SnapshotPolicyVO.class); + Mockito.when(policyVO.getVolumeId()).thenReturn(volumeId); + Mockito.when(snapshotPolicyDao.findById(policyId)).thenReturn(policyVO); + + // Volume doesn't exist at all (even when including removed) + Mockito.when(volumeDao.findByIdIncludingRemoved(volumeId)).thenReturn(null); + + snapshotManager.deleteSnapshotPolicies(cmd); + } + + @Test(expected = InvalidParameterValueException.class) + public void testDeleteSnapshotPoliciesManualPolicyId() { + DeleteSnapshotPoliciesCmd cmd = Mockito.mock(DeleteSnapshotPoliciesCmd.class); + Mockito.when(cmd.getId()).thenReturn(Snapshot.MANUAL_POLICY_ID); + Mockito.when(cmd.getIds()).thenReturn(null); + + snapshotManager.deleteSnapshotPolicies(cmd); + } } diff --git a/server/src/test/java/com/cloud/template/HypervisorTemplateAdapterTest.java b/server/src/test/java/com/cloud/template/HypervisorTemplateAdapterTest.java index 2a6d7af434a..e2a97be469f 100644 --- a/server/src/test/java/com/cloud/template/HypervisorTemplateAdapterTest.java +++ b/server/src/test/java/com/cloud/template/HypervisorTemplateAdapterTest.java @@ -49,7 +49,6 @@ import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.framework.events.Event; import org.apache.cloudstack.framework.events.EventDistributor; import org.apache.cloudstack.framework.messagebus.MessageBus; -import org.apache.cloudstack.secstorage.heuristics.HeuristicType; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import org.apache.cloudstack.storage.heuristics.HeuristicRuleHelper; @@ -339,7 +338,7 @@ public class HypervisorTemplateAdapterTest { Mockito.when(templateProfileMock.getZoneIdList()).thenReturn(zoneIds); Mockito.doReturn(null).when(_adapter).getImageStoresThrowsExceptionIfNotFound(Mockito.any(Long.class), Mockito.any(TemplateProfile.class)); - Mockito.doReturn(null).when(_adapter).verifyHeuristicRulesForZone(Mockito.any(VMTemplateVO.class), Mockito.anyLong()); + Mockito.doReturn(null).when(_templateMgr).verifyHeuristicRulesForZone(Mockito.any(VMTemplateVO.class), Mockito.anyLong()); Mockito.doNothing().when(_adapter).standardImageStoreAllocation(Mockito.isNull(), Mockito.any(VMTemplateVO.class)); _adapter.createTemplateWithinZones(templateProfileMock, vmTemplateVOMock); @@ -355,7 +354,7 @@ public class HypervisorTemplateAdapterTest { Mockito.when(templateProfileMock.getZoneIdList()).thenReturn(zoneIds); Mockito.doReturn(null).when(_adapter).getImageStoresThrowsExceptionIfNotFound(Mockito.any(Long.class), Mockito.any(TemplateProfile.class)); - Mockito.doReturn(null).when(_adapter).verifyHeuristicRulesForZone(Mockito.any(VMTemplateVO.class), Mockito.anyLong()); + Mockito.doReturn(null).when(_templateMgr).verifyHeuristicRulesForZone(Mockito.any(VMTemplateVO.class), Mockito.anyLong()); Mockito.doNothing().when(_adapter).standardImageStoreAllocation(Mockito.isNull(), Mockito.any(VMTemplateVO.class)); _adapter.createTemplateWithinZones(templateProfileMock, vmTemplateVOMock); @@ -371,7 +370,7 @@ public class HypervisorTemplateAdapterTest { List zoneIds = List.of(1L); Mockito.when(templateProfileMock.getZoneIdList()).thenReturn(zoneIds); - Mockito.doReturn(dataStoreMock).when(_adapter).verifyHeuristicRulesForZone(Mockito.any(VMTemplateVO.class), Mockito.anyLong()); + Mockito.doReturn(dataStoreMock).when(_templateMgr).verifyHeuristicRulesForZone(Mockito.any(VMTemplateVO.class), Mockito.anyLong()); Mockito.doNothing().when(_adapter).validateSecondaryStorageAndCreateTemplate(Mockito.any(List.class), Mockito.any(VMTemplateVO.class), Mockito.isNull()); _adapter.createTemplateWithinZones(templateProfileMock, vmTemplateVOMock); @@ -409,26 +408,6 @@ public class HypervisorTemplateAdapterTest { _adapter.getImageStoresThrowsExceptionIfNotFound(zoneId, templateProfileMock); } - @Test - public void verifyHeuristicRulesForZoneTestTemplateIsISOFormatShouldCheckForISOHeuristicType() { - VMTemplateVO vmTemplateVOMock = Mockito.mock(VMTemplateVO.class); - - Mockito.when(vmTemplateVOMock.getFormat()).thenReturn(ImageFormat.ISO); - _adapter.verifyHeuristicRulesForZone(vmTemplateVOMock, 1L); - - Mockito.verify(heuristicRuleHelperMock, Mockito.times(1)).getImageStoreIfThereIsHeuristicRule(1L, HeuristicType.ISO, vmTemplateVOMock); - } - - @Test - public void verifyHeuristicRulesForZoneTestTemplateNotISOFormatShouldCheckForTemplateHeuristicType() { - VMTemplateVO vmTemplateVOMock = Mockito.mock(VMTemplateVO.class); - - Mockito.when(vmTemplateVOMock.getFormat()).thenReturn(ImageFormat.QCOW2); - _adapter.verifyHeuristicRulesForZone(vmTemplateVOMock, 1L); - - Mockito.verify(heuristicRuleHelperMock, Mockito.times(1)).getImageStoreIfThereIsHeuristicRule(1L, HeuristicType.TEMPLATE, vmTemplateVOMock); - } - @Test public void isZoneAndImageStoreAvailableTestZoneIdIsNullShouldReturnFalse() { DataStore dataStoreMock = Mockito.mock(DataStore.class); diff --git a/server/src/test/java/com/cloud/template/TemplateManagerImplTest.java b/server/src/test/java/com/cloud/template/TemplateManagerImplTest.java index 819694a226b..7bdc1e7c604 100755 --- a/server/src/test/java/com/cloud/template/TemplateManagerImplTest.java +++ b/server/src/test/java/com/cloud/template/TemplateManagerImplTest.java @@ -24,6 +24,7 @@ import com.cloud.api.query.dao.SnapshotJoinDao; import com.cloud.api.query.dao.UserVmJoinDao; import com.cloud.configuration.Resource; import com.cloud.dc.dao.DataCenterDao; +import com.cloud.deployasis.dao.TemplateDeployAsIsDetailsDao; import com.cloud.domain.dao.DomainDao; import com.cloud.event.dao.UsageEventDao; import com.cloud.exception.InvalidParameterValueException; @@ -207,6 +208,8 @@ public class TemplateManagerImplTest { VnfTemplateManager vnfTemplateManager; @Inject SnapshotJoinDao snapshotJoinDao; + @Inject + TemplateDeployAsIsDetailsDao templateDeployAsIsDetailsDao; @Inject HeuristicRuleHelper heuristicRuleHelperMock; @@ -753,6 +756,26 @@ public class TemplateManagerImplTest { Assert.assertNull(type); } + @Test + public void verifyHeuristicRulesForZoneTestTemplateIsISOFormatShouldCheckForISOHeuristicType() { + VMTemplateVO vmTemplateVOMock = Mockito.mock(VMTemplateVO.class); + + Mockito.when(vmTemplateVOMock.getFormat()).thenReturn(Storage.ImageFormat.ISO); + templateManager.verifyHeuristicRulesForZone(vmTemplateVOMock, 1L); + + Mockito.verify(heuristicRuleHelperMock, Mockito.times(1)).getImageStoreIfThereIsHeuristicRule(1L, HeuristicType.ISO, vmTemplateVOMock); + } + + @Test + public void verifyHeuristicRulesForZoneTestTemplateNotISOFormatShouldCheckForTemplateHeuristicType() { + VMTemplateVO vmTemplateVOMock = Mockito.mock(VMTemplateVO.class); + + Mockito.when(vmTemplateVOMock.getFormat()).thenReturn(Storage.ImageFormat.QCOW2); + templateManager.verifyHeuristicRulesForZone(vmTemplateVOMock, 1L); + + Mockito.verify(heuristicRuleHelperMock, Mockito.times(1)).getImageStoreIfThereIsHeuristicRule(1L, HeuristicType.TEMPLATE, vmTemplateVOMock); + } + @Configuration @ComponentScan(basePackageClasses = {TemplateManagerImpl.class}, includeFilters = {@ComponentScan.Filter(value = TestConfiguration.Library.class, type = FilterType.CUSTOM)}, @@ -959,6 +982,11 @@ public class TemplateManagerImplTest { return Mockito.mock(VnfTemplateManager.class); } + @Bean + public TemplateDeployAsIsDetailsDao templateDeployAsIsDetailsDao() { + return Mockito.mock(TemplateDeployAsIsDetailsDao.class); + } + @Bean public SnapshotHelper snapshotHelper() { return Mockito.mock(SnapshotHelper.class); diff --git a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java index 1ec141a8be1..4edafb3a05a 100644 --- a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java +++ b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java @@ -40,8 +40,10 @@ import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; +import java.lang.reflect.Field; import java.text.SimpleDateFormat; import java.time.LocalDateTime; import java.time.ZoneOffset; @@ -59,8 +61,6 @@ import java.util.Map; import java.util.TimeZone; import java.util.UUID; -import com.cloud.domain.Domain; -import com.cloud.storage.dao.SnapshotPolicyDao; import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.acl.SecurityChecker; import org.apache.cloudstack.api.ApiCommandResourceType; @@ -88,6 +88,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore; import org.apache.cloudstack.engine.subsystem.api.storage.Scope; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; +import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.template.VnfTemplateManager; @@ -117,6 +118,7 @@ import com.cloud.deploy.DataCenterDeployment; import com.cloud.deploy.DeployDestination; import com.cloud.deploy.DeploymentPlanner; import com.cloud.deploy.DeploymentPlanningManager; +import com.cloud.domain.Domain; import com.cloud.domain.DomainVO; import com.cloud.domain.dao.DomainDao; import com.cloud.event.ActionEventUtils; @@ -142,9 +144,9 @@ import com.cloud.network.dao.LoadBalancerVMMapDao; import com.cloud.network.dao.LoadBalancerVMMapVO; import com.cloud.network.dao.NetworkDao; import com.cloud.network.dao.NetworkVO; -import com.cloud.network.element.UserDataServiceProvider; import com.cloud.network.dao.PhysicalNetworkDao; import com.cloud.network.dao.PhysicalNetworkVO; +import com.cloud.network.element.UserDataServiceProvider; import com.cloud.network.guru.NetworkGuru; import com.cloud.network.rules.FirewallRuleVO; import com.cloud.network.rules.PortForwardingRule; @@ -172,6 +174,7 @@ import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.DiskOfferingDao; import com.cloud.storage.dao.GuestOSDao; import com.cloud.storage.dao.SnapshotDao; +import com.cloud.storage.dao.SnapshotPolicyDao; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.template.VirtualMachineTemplate; @@ -467,6 +470,24 @@ public class UserVmManagerImplTest { Class expectedInvalidParameterValueException = InvalidParameterValueException.class; Class expectedCloudRuntimeException = CloudRuntimeException.class; + private Map originalConfigValues = new HashMap<>(); + + + private void updateDefaultConfigValue(final ConfigKey configKey, final Object o, boolean revert) { + try { + final String name = "_defaultValue"; + Field f = ConfigKey.class.getDeclaredField(name); + f.setAccessible(true); + String stringVal = String.valueOf(o); + if (!revert) { + originalConfigValues.put(configKey, f.get(configKey)); + } + f.set(configKey, stringVal); + } catch (IllegalAccessException | NoSuchFieldException e) { + Assert.fail("Failed to mock config " + configKey.key() + " value due to " + e.getMessage()); + } + } + @Before public void beforeTest() { userVmManagerImpl.resourceLimitService = resourceLimitMgr; @@ -496,6 +517,9 @@ public class UserVmManagerImplTest { public void afterTest() { CallContext.unregister(); unmanagedVMsManagerMockedStatic.close(); + for (Map.Entry entry : originalConfigValues.entrySet()) { + updateDefaultConfigValue(entry.getKey(), entry.getValue(), true); + } } @Test @@ -1136,7 +1160,7 @@ public class UserVmManagerImplTest { ReflectionTestUtils.setField(deployVMCmd, "serviceOfferingId", serviceOfferingId); deployVMCmd._accountService = accountService; - when(accountService.finalyzeAccountId(nullable(String.class), nullable(Long.class), nullable(Long.class), eq(true))).thenReturn(accountId); + when(accountService.finalizeAccountId(nullable(String.class), nullable(Long.class), nullable(Long.class), eq(true))).thenReturn(accountId); when(accountService.getActiveAccountById(accountId)).thenReturn(account); when(entityManager.findById(DataCenter.class, zoneId)).thenReturn(_dcMock); when(entityManager.findById(ServiceOffering.class, serviceOfferingId)).thenReturn(serviceOffering); @@ -1153,7 +1177,7 @@ public class UserVmManagerImplTest { ReflectionTestUtils.setField(deployVMCmd, "serviceOfferingId", serviceOfferingId); deployVMCmd._accountService = accountService; - when(accountService.finalyzeAccountId(nullable(String.class), nullable(Long.class), nullable(Long.class), eq(true))).thenReturn(accountId); + when(accountService.finalizeAccountId(nullable(String.class), nullable(Long.class), nullable(Long.class), eq(true))).thenReturn(accountId); when(accountService.getActiveAccountById(accountId)).thenReturn(account); when(entityManager.findById(DataCenter.class, zoneId)).thenReturn(_dcMock); when(entityManager.findById(ServiceOffering.class, serviceOfferingId)).thenReturn(serviceOffering); @@ -1164,7 +1188,7 @@ public class UserVmManagerImplTest { when(templateMock.isDeployAsIs()).thenReturn(false); when(templateMock.getFormat()).thenReturn(Storage.ImageFormat.QCOW2); when(templateMock.getUserDataId()).thenReturn(null); - Mockito.doNothing().when(vnfTemplateManager).validateVnfApplianceNics(any(), nullable(List.class)); + Mockito.doNothing().when(vnfTemplateManager).validateVnfApplianceNics(any(), nullable(List.class), nullable(Map.class)); ServiceOfferingJoinVO svcOfferingMock = Mockito.mock(ServiceOfferingJoinVO.class); when(serviceOfferingJoinDao.findById(anyLong())).thenReturn(svcOfferingMock); @@ -1176,7 +1200,7 @@ public class UserVmManagerImplTest { UserVm result = userVmManagerImpl.createVirtualMachine(deployVMCmd); assertEquals(userVmVoMock, result); - Mockito.verify(vnfTemplateManager).validateVnfApplianceNics(templateMock, null); + Mockito.verify(vnfTemplateManager).validateVnfApplianceNics(templateMock, null, Collections.emptyMap()); Mockito.verify(userVmManagerImpl).createBasicSecurityGroupVirtualMachine(any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), nullable(Boolean.class), any(), any(), any(), any(), any(), any(), any(), eq(true), any(), any(), any()); @@ -1409,7 +1433,7 @@ public class UserVmManagerImplTest { ReflectionTestUtils.setField(deployVMCmd, "serviceOfferingId", serviceOfferingId); deployVMCmd._accountService = accountService; - when(accountService.finalyzeAccountId(nullable(String.class), nullable(Long.class), nullable(Long.class), eq(true))).thenReturn(accountId); + when(accountService.finalizeAccountId(nullable(String.class), nullable(Long.class), nullable(Long.class), eq(true))).thenReturn(accountId); when(accountService.getActiveAccountById(accountId)).thenReturn(account); when(entityManager.findById(DataCenter.class, zoneId)).thenReturn(_dcMock); when(entityManager.findById(ServiceOffering.class, serviceOfferingId)).thenReturn(serviceOffering); @@ -1420,7 +1444,7 @@ public class UserVmManagerImplTest { when(templateMock.isDeployAsIs()).thenReturn(false); when(templateMock.getFormat()).thenReturn(Storage.ImageFormat.QCOW2); when(templateMock.getUserDataId()).thenReturn(null); - Mockito.doNothing().when(vnfTemplateManager).validateVnfApplianceNics(any(), nullable(List.class)); + Mockito.doNothing().when(vnfTemplateManager).validateVnfApplianceNics(any(), nullable(List.class), nullable(Map.class)); ServiceOfferingJoinVO svcOfferingMock = Mockito.mock(ServiceOfferingJoinVO.class); when(serviceOfferingJoinDao.findById(anyLong())).thenReturn(svcOfferingMock); @@ -3303,7 +3327,7 @@ public class UserVmManagerImplTest { CreateVMFromBackupCmd cmd = new CreateVMFromBackupCmd(); cmd._accountService = accountService; cmd._entityMgr = entityManager; - when(accountService.finalyzeAccountId(nullable(String.class), nullable(Long.class), nullable(Long.class), eq(true))).thenReturn(accountId); + when(accountService.finalizeAccountId(nullable(String.class), nullable(Long.class), nullable(Long.class), eq(true))).thenReturn(accountId); when(accountService.getActiveAccountById(accountId)).thenReturn(account); ReflectionTestUtils.setField(cmd, "serviceOfferingId", serviceOfferingId); @@ -3474,7 +3498,7 @@ public class UserVmManagerImplTest { CreateVMFromBackupCmd cmd = new CreateVMFromBackupCmd(); cmd._accountService = accountService; cmd._entityMgr = entityManager; - when(accountService.finalyzeAccountId(nullable(String.class), nullable(Long.class), nullable(Long.class), eq(true))).thenReturn(accountId); + when(accountService.finalizeAccountId(nullable(String.class), nullable(Long.class), nullable(Long.class), eq(true))).thenReturn(accountId); when(accountService.getActiveAccountById(accountId)).thenReturn(account); ReflectionTestUtils.setField(cmd, "serviceOfferingId", serviceOfferingId); @@ -3863,7 +3887,7 @@ public class UserVmManagerImplTest { ReflectionTestUtils.setField(deployVMCmd, "volumeId", volumeId); deployVMCmd._accountService = accountService; - when(accountService.finalyzeAccountId(nullable(String.class), nullable(Long.class), nullable(Long.class), eq(true))).thenReturn(accountId); + when(accountService.finalizeAccountId(nullable(String.class), nullable(Long.class), nullable(Long.class), eq(true))).thenReturn(accountId); when(accountService.getActiveAccountById(accountId)).thenReturn(account); when(entityManager.findById(DataCenter.class, zoneId)).thenReturn(_dcMock); when(entityManager.findById(ServiceOffering.class, serviceOfferingId)).thenReturn(serviceOffering); @@ -3880,7 +3904,7 @@ public class UserVmManagerImplTest { when(templateMock.isDeployAsIs()).thenReturn(false); when(templateMock.getFormat()).thenReturn(Storage.ImageFormat.QCOW2); when(templateMock.getUserDataId()).thenReturn(null); - Mockito.doNothing().when(vnfTemplateManager).validateVnfApplianceNics(any(), nullable(List.class)); + Mockito.doNothing().when(vnfTemplateManager).validateVnfApplianceNics(any(), nullable(List.class), any()); when(_dcMock.isLocalStorageEnabled()).thenReturn(false); when(_dcMock.getNetworkType()).thenReturn(DataCenter.NetworkType.Basic); Mockito.doReturn(userVmVoMock).when(userVmManagerImpl).createBasicSecurityGroupVirtualMachine(any(), any(), any(), any(), any(), any(), any(), @@ -3899,7 +3923,7 @@ public class UserVmManagerImplTest { ReflectionTestUtils.setField(deployVMCmd, "snapshotId", snashotId); deployVMCmd._accountService = accountService; - when(accountService.finalyzeAccountId(nullable(String.class), nullable(Long.class), nullable(Long.class), eq(true))).thenReturn(accountId); + when(accountService.finalizeAccountId(nullable(String.class), nullable(Long.class), nullable(Long.class), eq(true))).thenReturn(accountId); when(accountService.getActiveAccountById(accountId)).thenReturn(account); when(entityManager.findById(DataCenter.class, zoneId)).thenReturn(_dcMock); when(entityManager.findById(ServiceOffering.class, serviceOfferingId)).thenReturn(serviceOffering); @@ -3918,7 +3942,7 @@ public class UserVmManagerImplTest { when(templateMock.isDeployAsIs()).thenReturn(false); when(templateMock.getFormat()).thenReturn(Storage.ImageFormat.QCOW2); when(templateMock.getUserDataId()).thenReturn(null); - Mockito.doNothing().when(vnfTemplateManager).validateVnfApplianceNics(any(), nullable(List.class)); + Mockito.doNothing().when(vnfTemplateManager).validateVnfApplianceNics(any(), nullable(List.class), any()); when(_dcMock.isLocalStorageEnabled()).thenReturn(false); when(_dcMock.getNetworkType()).thenReturn(DataCenter.NetworkType.Basic); Mockito.doReturn(userVmVoMock).when(userVmManagerImpl).createBasicSecurityGroupVirtualMachine(any(), any(), any(), any(), any(), any(), any(), @@ -3937,7 +3961,7 @@ public class UserVmManagerImplTest { CreateVMFromBackupCmd cmd = new CreateVMFromBackupCmd(); cmd._accountService = accountService; cmd._entityMgr = entityManager; - when(accountService.finalyzeAccountId(nullable(String.class), nullable(Long.class), nullable(Long.class), eq(true))).thenReturn(accountId); + when(accountService.finalizeAccountId(nullable(String.class), nullable(Long.class), nullable(Long.class), eq(true))).thenReturn(accountId); when(accountService.getActiveAccountById(accountId)).thenReturn(account); ReflectionTestUtils.setField(cmd, "serviceOfferingId", serviceOfferingId); @@ -4003,7 +4027,7 @@ public class UserVmManagerImplTest { CreateVMFromBackupCmd cmd = new CreateVMFromBackupCmd(); cmd._accountService = accountService; cmd._entityMgr = entityManager; - when(accountService.finalyzeAccountId(nullable(String.class), nullable(Long.class), nullable(Long.class), eq(true))).thenReturn(accountId); + when(accountService.finalizeAccountId(nullable(String.class), nullable(Long.class), nullable(Long.class), eq(true))).thenReturn(accountId); when(accountService.getActiveAccountById(accountId)).thenReturn(account); ReflectionTestUtils.setField(cmd, "serviceOfferingId", serviceOfferingId); @@ -4178,4 +4202,49 @@ public class UserVmManagerImplTest { verify(userVmDao, times(1)).releaseFromLockTable(vmId); } + @Test + public void updateVmExtraConfigCleansUpWhenCleanupFlagIsTrue() { + UserVmVO userVm = mock(UserVmVO.class); + when(userVm.getUuid()).thenReturn("test-uuid"); + when(userVm.getId()).thenReturn(1L); + + userVmManagerImpl.updateVmExtraConfig(userVm, "someConfig", true); + + verify(vmInstanceDetailsDao, times(1)).removeDetailsWithPrefix(1L, ApiConstants.EXTRA_CONFIG); + verifyNoMoreInteractions(vmInstanceDetailsDao); + } + + @Test + public void updateVmExtraConfigAddsConfigWhenValidAndEnabled() { + UserVmVO userVm = mock(UserVmVO.class); + when(userVm.getUuid()).thenReturn("test-uuid"); + when(userVm.getAccountId()).thenReturn(1L); + when(userVm.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM); + doNothing().when(userVmManagerImpl).persistExtraConfigKvm(anyString(), eq(userVm)); + updateDefaultConfigValue(UserVmManagerImpl.EnableAdditionalVmConfig, true, false); + + userVmManagerImpl.updateVmExtraConfig(userVm, "validConfig", false); + + verify(vmInstanceDetailsDao, never()).removeDetailsWithPrefix(anyLong(), anyString()); + verify(userVmManagerImpl, times(1)).addExtraConfig(userVm, "validConfig"); + } + + @Test(expected = InvalidParameterValueException.class) + public void updateVmExtraConfigThrowsExceptionWhenConfigDisabled() { + UserVmVO userVm = mock(UserVmVO.class); + when(userVm.getAccountId()).thenReturn(1L); + updateDefaultConfigValue(UserVmManagerImpl.EnableAdditionalVmConfig, false, false); + + userVmManagerImpl.updateVmExtraConfig(userVm, "validConfig", false); + } + + @Test + public void updateVmExtraConfigDoesNothingWhenExtraConfigIsBlank() { + UserVmVO userVm = mock(UserVmVO.class); + + userVmManagerImpl.updateVmExtraConfig(userVm, "", false); + + verify(vmInstanceDetailsDao, never()).removeDetailsWithPrefix(anyLong(), anyString()); + verify(userVmManagerImpl, never()).addExtraConfig(any(UserVmVO.class), anyString()); + } } diff --git a/server/src/test/java/com/cloud/vm/snapshot/VMSnapshotManagerTest.java b/server/src/test/java/com/cloud/vm/snapshot/VMSnapshotManagerTest.java index a0f09981a40..b696d743ac6 100644 --- a/server/src/test/java/com/cloud/vm/snapshot/VMSnapshotManagerTest.java +++ b/server/src/test/java/com/cloud/vm/snapshot/VMSnapshotManagerTest.java @@ -41,6 +41,7 @@ import com.cloud.storage.dao.SnapshotDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.user.Account; import com.cloud.user.AccountManager; +import com.cloud.user.AccountVO; import com.cloud.user.dao.AccountDao; import com.cloud.user.dao.UserDao; import com.cloud.uservm.UserVm; @@ -136,6 +137,8 @@ public class VMSnapshotManagerTest { VMSnapshotDetailsDao _vmSnapshotDetailsDao; @Mock UserVmManager _userVmManager; + @Mock + private AccountVO accountVOMock; private static final long TEST_VM_ID = 3L; private static final long SERVICE_OFFERING_ID = 1L; @@ -285,8 +288,12 @@ public class VMSnapshotManagerTest { @SuppressWarnings("unchecked") @Test(expected = CloudRuntimeException.class) public void testAllocVMSnapshotF4() throws ResourceAllocationException { + long accountId = 1L; List mockList = mock(List.class); when(mockList.size()).thenReturn(10); + when(_userVMDao.findById(TEST_VM_ID)).thenReturn(vmMock); + when(userVm.getAccountId()).thenReturn(accountId); + when(_accountMgr.getAccount(accountId)).thenReturn(accountVOMock); when(_vmSnapshotDao.findByVm(TEST_VM_ID)).thenReturn(mockList); _vmSnapshotMgr.allocVMSnapshot(TEST_VM_ID, "", "", true); } @@ -295,8 +302,12 @@ public class VMSnapshotManagerTest { @SuppressWarnings("unchecked") @Test(expected = CloudRuntimeException.class) public void testAllocVMSnapshotF5() throws ResourceAllocationException { + long accountId = 1L; List mockList = mock(List.class); when(mockList.size()).thenReturn(1); + when(_userVMDao.findById(TEST_VM_ID)).thenReturn(vmMock); + when(userVm.getAccountId()).thenReturn(accountId); + when(_accountMgr.getAccount(accountId)).thenReturn(accountVOMock); when(_snapshotDao.listByInstanceId(TEST_VM_ID, Snapshot.State.Creating, Snapshot.State.CreatedOnPrimary, Snapshot.State.BackingUp)).thenReturn(mockList); _vmSnapshotMgr.allocVMSnapshot(TEST_VM_ID, "", "", true); } @@ -304,6 +315,10 @@ public class VMSnapshotManagerTest { // successful creation case @Test public void testCreateVMSnapshot() throws AgentUnavailableException, OperationTimedoutException, ResourceAllocationException, NoTransitionException { + long accountId = 1L; + when(_userVMDao.findById(TEST_VM_ID)).thenReturn(vmMock); + when(userVm.getAccountId()).thenReturn(accountId); + when(_accountMgr.getAccount(accountId)).thenReturn(accountVOMock); when(vmMock.getState()).thenReturn(State.Running); _vmSnapshotMgr.allocVMSnapshot(TEST_VM_ID, "", "", true); } diff --git a/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java b/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java index 93120673720..54d8d67b6f8 100644 --- a/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java +++ b/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java @@ -231,6 +231,13 @@ public class MockNetworkManagerImpl extends ManagerBase implements NetworkOrches return null; } + @Override + public Network createGuestNetwork(long networkOfferingId, String name, String displayText, Account owner, + PhysicalNetwork physicalNetwork, long zoneId, ACLType aclType, Pair vrIfaceMTUs) + throws InsufficientCapacityException, ConcurrentOperationException, ResourceAllocationException { + return null; + } + /* (non-Javadoc) * @see com.cloud.network.NetworkService#searchForNetworks(com.cloud.api.commands.ListNetworksCmd) */ diff --git a/server/src/test/java/org/apache/cloudstack/affinity/AffinityApiUnitTest.java b/server/src/test/java/org/apache/cloudstack/affinity/AffinityApiUnitTest.java index e27d9618c92..e8cc05c137e 100644 --- a/server/src/test/java/org/apache/cloudstack/affinity/AffinityApiUnitTest.java +++ b/server/src/test/java/org/apache/cloudstack/affinity/AffinityApiUnitTest.java @@ -190,7 +190,7 @@ public class AffinityApiUnitTest { @Test(expected = InvalidParameterValueException.class) public void deleteAffinityGroupInvalidIdName() throws ResourceInUseException { - when(_acctMgr.finalyzeAccountId("user", domainId, null, true)).thenReturn(200L); + when(_acctMgr.finalizeAccountId("user", domainId, null, true)).thenReturn(200L); when(_groupDao.findByAccountAndName(200L, "group1")).thenReturn(null); _affinityService.deleteAffinityGroup(null, "user", null, domainId, "group1"); } diff --git a/server/src/test/java/org/apache/cloudstack/affinity/AffinityGroupServiceImplTest.java b/server/src/test/java/org/apache/cloudstack/affinity/AffinityGroupServiceImplTest.java index 5657bd1f506..495561429ea 100644 --- a/server/src/test/java/org/apache/cloudstack/affinity/AffinityGroupServiceImplTest.java +++ b/server/src/test/java/org/apache/cloudstack/affinity/AffinityGroupServiceImplTest.java @@ -253,7 +253,7 @@ public class AffinityGroupServiceImplTest { @Test(expected = InvalidParameterValueException.class) public void deleteAffinityGroupInvalidIdName() throws ResourceInUseException { when(_acctMgr.finalizeOwner((Account)any(), anyString(), anyLong(), anyLong())).thenReturn(acct); - when(_acctMgr.finalyzeAccountId(ACCOUNT_NAME, DOMAIN_ID, null, true)).thenReturn(200L); + when(_acctMgr.finalizeAccountId(ACCOUNT_NAME, DOMAIN_ID, null, true)).thenReturn(200L); when(_groupDao.findByAccountAndName(200L, AFFINITY_GROUP_NAME)).thenReturn(null); _affinityService.deleteAffinityGroup(null, ACCOUNT_NAME, null, DOMAIN_ID, AFFINITY_GROUP_NAME); } diff --git a/server/src/test/java/org/apache/cloudstack/network/RoutedIpv4ManagerImplTest.java b/server/src/test/java/org/apache/cloudstack/network/RoutedIpv4ManagerImplTest.java index 81110dd3f53..737273e8379 100644 --- a/server/src/test/java/org/apache/cloudstack/network/RoutedIpv4ManagerImplTest.java +++ b/server/src/test/java/org/apache/cloudstack/network/RoutedIpv4ManagerImplTest.java @@ -340,7 +340,7 @@ public class RoutedIpv4ManagerImplTest { when(dataCenterIpv4GuestSubnetDao.findById(zoneSubnetId)).thenReturn(subnetVO); - when(accountManager.finalyzeAccountId(accountName, domainId, null, false)).thenReturn(accountId); + when(accountManager.finalizeAccountId(accountName, domainId, null, false)).thenReturn(accountId); when(accountManager.getAccount(accountId)).thenReturn(account); when(account.getDomainId()).thenReturn(domainId); when(dataCenterIpv4GuestSubnetDao.findById(zoneSubnetId)).thenReturn(subnetVO); @@ -911,7 +911,7 @@ public class RoutedIpv4ManagerImplTest { ReflectionTestUtils.setField(cmd,"projectId", null); when(bgpPeerDao.findById(bgpPeerId)).thenReturn(bgpPeer); - when(accountManager.finalyzeAccountId(accountName, domainId, null, false)).thenReturn(accountId); + when(accountManager.finalizeAccountId(accountName, domainId, null, false)).thenReturn(accountId); when(accountManager.getAccount(accountId)).thenReturn(account); when(account.getDomainId()).thenReturn(domainId); diff --git a/server/src/test/java/org/apache/cloudstack/storage/template/VnfTemplateManagerImplTest.java b/server/src/test/java/org/apache/cloudstack/storage/template/VnfTemplateManagerImplTest.java index c3fa0d62604..b9565ebb292 100644 --- a/server/src/test/java/org/apache/cloudstack/storage/template/VnfTemplateManagerImplTest.java +++ b/server/src/test/java/org/apache/cloudstack/storage/template/VnfTemplateManagerImplTest.java @@ -228,25 +228,25 @@ public class VnfTemplateManagerImplTest { @Test public void testValidateVnfApplianceNicsWithRequiredNics() { List networkIds = Arrays.asList(200L, 201L); - vnfTemplateManagerImpl.validateVnfApplianceNics(template, networkIds); + vnfTemplateManagerImpl.validateVnfApplianceNics(template, networkIds, null); } @Test public void testValidateVnfApplianceNicsWithAllNics() { List networkIds = Arrays.asList(200L, 201L, 202L); - vnfTemplateManagerImpl.validateVnfApplianceNics(template, networkIds); + vnfTemplateManagerImpl.validateVnfApplianceNics(template, networkIds, null); } @Test(expected = InvalidParameterValueException.class) public void testValidateVnfApplianceNicsWithEmptyList() { List networkIds = new ArrayList<>(); - vnfTemplateManagerImpl.validateVnfApplianceNics(template, networkIds); + vnfTemplateManagerImpl.validateVnfApplianceNics(template, networkIds, null); } @Test(expected = InvalidParameterValueException.class) public void testValidateVnfApplianceNicsWithMissingNetworkId() { List networkIds = Arrays.asList(200L); - vnfTemplateManagerImpl.validateVnfApplianceNics(template, networkIds); + vnfTemplateManagerImpl.validateVnfApplianceNics(template, networkIds, null); } @Test diff --git a/services/console-proxy/rdpconsole/src/main/java/common/Client.java b/services/console-proxy/rdpconsole/src/main/java/common/Client.java index 742f5c9f0cd..972d5d753e8 100644 --- a/services/console-proxy/rdpconsole/src/main/java/common/Client.java +++ b/services/console-proxy/rdpconsole/src/main/java/common/Client.java @@ -210,7 +210,6 @@ public class Client { public void runClient(String[] args) { try { - Protocol protocol = parseOptions(args); if (protocol == Protocol.NONE) return; @@ -299,21 +298,28 @@ public class Client { private Protocol parseOptions(String[] args) { String protocolName = (args.length > 0) ? args[0] : ""; - Protocol protocol = Protocol.NONE; + Protocol protocol; Option[] options; - if (protocolName.equals("vnc")) { - protocol = Protocol.VNC; - options = join(commonOptions, vncOptions); - } else if (protocolName.equals("rdp")) { - protocol = Protocol.RDP; - options = join(commonOptions, rdpOptions); - } else if (protocolName.equals("hyperv")) { - protocol = Protocol.HYPERV; - options = join(commonOptions, hyperVOptions); - } else { - help(); - return Protocol.NONE; + try { + protocol = Protocol.valueOf(protocolName); + } catch (IllegalArgumentException e) { + protocol = Protocol.NONE; + } + + switch (protocol) { + case VNC: + options = join(commonOptions, vncOptions); + break; + case RDP: + options = join(commonOptions, rdpOptions); + break; + case HYPERV: + options = join(commonOptions, hyperVOptions); + break; + default: + help(); + return Protocol.NONE; } // Parse all options for given protocol diff --git a/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java b/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java index 2ee1dcb37da..9d4c7311159 100644 --- a/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java +++ b/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java @@ -1229,8 +1229,10 @@ public class SecondaryStorageManagerImpl extends ManagerBase implements Secondar if (dc.getDns2() != null) { buf.append(" dns2=").append(dc.getDns2()); } - String nfsVersion = imageStoreDetailsUtil != null ? imageStoreDetailsUtil.getNfsVersion(secStores.get(0).getId()) : null; - buf.append(" nfsVersion=").append(nfsVersion); + String nfsVersion = imageStoreDetailsUtil.getNfsVersion(secStores.get(0).getId()); + if (StringUtils.isNotBlank(nfsVersion)) { + buf.append(" nfsVersion=").append(nfsVersion); + } buf.append(" keystore_password=").append(VirtualMachineGuru.getEncodedString(PasswordGenerator.generateRandomPassword(16))); if (SystemVmEnableUserData.valueIn(dc.getId())) { @@ -1250,7 +1252,7 @@ public class SecondaryStorageManagerImpl extends ManagerBase implements Secondar logger.debug(String.format("Boot args for machine profile [%s]: [%s].", profile.toString(), bootArgs)); } - boolean useHttpsToUpload = BooleanUtils.toBooleanDefaultIfNull(VolumeApiService.UseHttpsToUpload.value(), true); + boolean useHttpsToUpload = VolumeApiService.UseHttpsToUpload.valueIn(dc.getId()); logger.debug(String.format("Setting UseHttpsToUpload config on cmdline with [%s] value.", useHttpsToUpload)); buf.append(" useHttpsToUpload=").append(useHttpsToUpload); diff --git a/setup/dev/s3.cfg b/setup/dev/s3.cfg index de28e5b2698..ce414f584cf 100644 --- a/setup/dev/s3.cfg +++ b/setup/dev/s3.cfg @@ -1,20 +1,19 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# http://www.apache.org/licenses/LICENSE-2.0 # +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. # TODO: Change ACCESS_KEY/ SECRET_KEY to your credentials on the object store diff --git a/systemvm/debian/etc/apache2/vhost.template b/systemvm/debian/etc/apache2/vhost.template index 7f6a5146099..d1c7a2960ee 100644 --- a/systemvm/debian/etc/apache2/vhost.template +++ b/systemvm/debian/etc/apache2/vhost.template @@ -39,8 +39,8 @@ Allow from 127.0.0.0/255.0.0.0 ::1/128 - # Include HTTP configuration **IF SET** - IncludeOptional /etc/apache2/http.conf + # Include CORS configuration **IF SET** + IncludeOptional /etc/apache2/cors.conf @@ -86,8 +86,8 @@ Allow from 127.0.0.0/255.0.0.0 ::1/128 - # Include HTTPS configuration **IF SET** - IncludeOptional /etc/apache2/https.conf + # Include CORS configuration **IF SET** + IncludeOptional /etc/apache2/cors.conf # SSL Engine Switch: # Enable/Disable SSL for this virtual host. diff --git a/systemvm/debian/etc/haproxy/haproxy.cfg b/systemvm/debian/etc/haproxy/haproxy.cfg index 21964f297c2..68a4cd7cd58 100644 --- a/systemvm/debian/etc/haproxy/haproxy.cfg +++ b/systemvm/debian/etc/haproxy/haproxy.cfg @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + global log 127.0.0.1:3914 local0 info chroot /var/lib/haproxy diff --git a/systemvm/debian/opt/cloud/bin/configure.py b/systemvm/debian/opt/cloud/bin/configure.py index c3c91d90c5a..bf48be66694 100755 --- a/systemvm/debian/opt/cloud/bin/configure.py +++ b/systemvm/debian/opt/cloud/bin/configure.py @@ -1233,17 +1233,21 @@ class CsRemoteAccessVpn(CsDataBag): CsHelper.start_if_stopped("ipsec") logging.debug("Remote accessvpn data bag %s", self.dbag) + config_changed = False if not self.config.has_public_network(): interface = self.config.address().get_guest_if_by_network_id() if interface: - self.configure_l2tpIpsec(interface.get_ip(), self.dbag[public_ip]) + config_changed = self.configure_l2tpIpsec(interface.get_ip(), self.dbag[public_ip]) self.remoteaccessvpn_iptables(interface.get_device(), interface.get_ip(), self.dbag[public_ip]) else: - self.configure_l2tpIpsec(public_ip, self.dbag[public_ip]) + config_changed = self.configure_l2tpIpsec(public_ip, self.dbag[public_ip]) self.remoteaccessvpn_iptables(self.dbag[public_ip]['public_interface'], public_ip, self.dbag[public_ip]) CsHelper.execute("ipsec update") - CsHelper.execute("systemctl start xl2tpd") + if config_changed: + CsHelper.execute("systemctl restart xl2tpd") + else: + CsHelper.execute("systemctl start xl2tpd") CsHelper.execute("ipsec rereadsecrets") else: logging.debug("Disabling remote access vpn .....") @@ -1266,21 +1270,23 @@ class CsRemoteAccessVpn(CsDataBag): l2tpfile = CsFile(l2tpconffile) l2tpfile.addeq(" left=%s" % left) l2tpfile.addeq(" leftid=%s" % obj['vpn_server_ip']) - l2tpfile.commit() + l2tp_changed = l2tpfile.commit() secret = CsFile(vpnsecretfilte) secret.empty() secret.addeq(": PSK \"%s\"" % (psk)) - secret.commit() + secret_changed = secret.commit() xl2tpdconf = CsFile(xl2tpdconffile) xl2tpdconf.addeq("ip range = %s" % iprange) xl2tpdconf.addeq("local ip = %s" % localip) - xl2tpdconf.commit() + xl2tpd_changed = xl2tpdconf.commit() xl2tpoptions = CsFile(xl2tpoptionsfile) xl2tpoptions.search("ms-dns ", "ms-dns %s" % localip) - xl2tpoptions.commit() + xl2tpoptions_changed = xl2tpoptions.commit() + + return l2tp_changed or secret_changed or xl2tpd_changed or xl2tpoptions_changed def remoteaccessvpn_iptables(self, publicdev, publicip, obj): localcidr = obj['local_cidr'] diff --git a/systemvm/debian/opt/cloud/bin/cs/CsDhcp.py b/systemvm/debian/opt/cloud/bin/cs/CsDhcp.py index e15714af212..a2309067289 100755 --- a/systemvm/debian/opt/cloud/bin/cs/CsDhcp.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsDhcp.py @@ -110,7 +110,7 @@ class CsDhcp(CsDataBag): if gn.get_dns() and device: sline = "dhcp-option=tag:interface-%s-%s,6" % (device, idx) dns_list = [x for x in gn.get_dns() if x] - if (self.config.is_vpc() or self.config.is_router()) and ('is_vr_guest_gateway' in gn.data and gn.data['is_vr_guest_gateway']): + if self.config.is_vpc() and not gn.is_vr_guest_gateway(): if gateway in dns_list: dns_list.remove(gateway) if gn.data['router_guest_ip'] != ip: diff --git a/systemvm/debian/opt/cloud/bin/setup/common.sh b/systemvm/debian/opt/cloud/bin/setup/common.sh index 85b78ee16ae..ef1576ab588 100755 --- a/systemvm/debian/opt/cloud/bin/setup/common.sh +++ b/systemvm/debian/opt/cloud/bin/setup/common.sh @@ -924,9 +924,6 @@ parse_cmd_line() { privateMtu) export PRIVATEMTU=$VALUE ;; - useHttpsToUpload) - export USEHTTPS=$VALUE - ;; vncport) export VNCPORT=$VALUE ;; diff --git a/systemvm/debian/opt/cloud/bin/setup/secstorage.sh b/systemvm/debian/opt/cloud/bin/setup/secstorage.sh index b8ed7b54311..5baed567f8f 100755 --- a/systemvm/debian/opt/cloud/bin/setup/secstorage.sh +++ b/systemvm/debian/opt/cloud/bin/setup/secstorage.sh @@ -50,33 +50,14 @@ setup_secstorage() { a2enmod proxy_http a2enmod headers - if [ -z $USEHTTPS ] | $USEHTTPS ; then - if [ -f /etc/apache2/http.conf ]; then - rm -rf /etc/apache2/http.conf - fi - cat >/etc/apache2/https.conf </etc/apache2/http.conf </etc/apache2/cors.conf < org.codehaus.mojo properties-maven-plugin - 1.0-alpha-2 + 1.2.1 initialize diff --git a/tools/devcloud4/advanced/marvin.cfg b/tools/devcloud4/advanced/marvin.cfg index 222dc65d045..7b6e656e620 100644 --- a/tools/devcloud4/advanced/marvin.cfg +++ b/tools/devcloud4/advanced/marvin.cfg @@ -1,21 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# http://www.apache.org/licenses/LICENSE-2.0 # +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. { "zones": [ diff --git a/tools/devcloud4/basic/marvin.cfg b/tools/devcloud4/basic/marvin.cfg index 1c8ee547b26..9b7d73c381b 100644 --- a/tools/devcloud4/basic/marvin.cfg +++ b/tools/devcloud4/basic/marvin.cfg @@ -1,21 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at # -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# http://www.apache.org/licenses/LICENSE-2.0 # +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. { "zones": [ diff --git a/tools/devcloud4/pom.xml b/tools/devcloud4/pom.xml index 1af63b439ad..385b49ad88c 100644 --- a/tools/devcloud4/pom.xml +++ b/tools/devcloud4/pom.xml @@ -56,7 +56,7 @@ org.codehaus.mojo properties-maven-plugin - 1.0-alpha-2 + 1.2.1 initialize diff --git a/tools/docker/Dockerfile.s390x b/tools/docker/Dockerfile.s390x new file mode 100644 index 00000000000..7e8c49fb386 --- /dev/null +++ b/tools/docker/Dockerfile.s390x @@ -0,0 +1,90 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# CloudStack-simulator build + +FROM ubuntu:22.04 + +LABEL Vendor="Apache.org" License="ApacheV2" Version="4.23.0.0-SNAPSHOT" Author="Apache CloudStack " + +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt-get -y update && apt-get install -y \ + genisoimage \ + libffi-dev \ + libssl-dev \ + curl \ + gcc-10 \ + git \ + sudo \ + ipmitool \ + iproute2 \ + maven \ + openjdk-11-jdk \ + python3-dev \ + python-is-python3 \ + python3-setuptools \ + python3-pip \ + python3-mysql.connector \ + # Required on s390x as pre-built wheels for bcrypt, cryptography, and cffi are unavailable, necessitating source builds. + python3-bcrypt \ + python3-cryptography \ + python3-cffi \ + supervisor + +RUN apt-get install -qqy mysql-server && \ + apt-get clean all && \ + mkdir -p /var/run/mysqld; \ + chown mysql /var/run/mysqld + +RUN echo '''sql_mode = "STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION"''' >> /etc/mysql/mysql.conf.d/mysqld.cnf + +COPY tools/docker/supervisord.conf /etc/supervisor/conf.d/supervisord.conf +COPY . ./root +WORKDIR /root + +RUN mvn -Pdeveloper -Dsimulator -DskipTests clean install + +RUN find /var/lib/mysql -type f -exec touch {} \; && \ + (/usr/bin/mysqld_safe &) && \ + sleep 5; \ + mysql -e "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password by ''" --connect-expired-password; \ + mvn -Pdeveloper -pl developer -Ddeploydb; \ + mvn -Pdeveloper -pl developer -Ddeploydb-simulator; \ + MARVIN_FILE=`find /root/tools/marvin/dist/ -name "Marvin*.tar.gz"`; \ + rm -rf /usr/bin/s390x-linux-gnu-gcc && \ + ln -s /usr/bin/gcc-10 /usr/bin/s390x-linux-gnu-gcc && \ + pip3 install maturin && \ + pip3 install $MARVIN_FILE + +RUN apt-get install -y nodejs npm build-essential python3 g++ make + +RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.4/install.sh | bash && \ + . /root/.nvm/nvm.sh && \ + nvm install 16 && \ + nvm use 16 && \ + NVM_BIN="$(dirname "$(nvm which node)")" && \ + ln -sf "$NVM_BIN/node" /usr/local/bin/node && \ + ln -sf "$NVM_BIN/npm" /usr/local/bin/npm && \ + cd ui && npm rebuild node-sass && npm install + + +VOLUME /var/lib/mysql + +EXPOSE 8080 8096 5050 + +CMD ["/usr/bin/supervisord"] diff --git a/tools/marvin/README.md b/tools/marvin/README.md new file mode 100644 index 00000000000..0d43605fc0a --- /dev/null +++ b/tools/marvin/README.md @@ -0,0 +1,29 @@ + +# Marvin + +Marvin is the Apache CloudStack Python client written for running smoke tests. + +## Overview + +Marvin provides a Python-based client for Apache CloudStack. It offers utilities for testing and interacting with CloudStack deployments. + +## License + +Licensed under the Apache License, Version 2.0. See the LICENSE.txt file for details. diff --git a/tools/marvin/marvin/lib/base.py b/tools/marvin/marvin/lib/base.py index bbec2027bdc..2405d54162a 100755 --- a/tools/marvin/marvin/lib/base.py +++ b/tools/marvin/marvin/lib/base.py @@ -1725,11 +1725,12 @@ class Template: # If template is ready, # template.status = Download Complete # Downloading - x% Downloaded + # Processing - Initial status # Error - Any other string if template.status == 'Download Complete' and template.isready: return - elif 'Downloaded' in template.status: + elif 'Downloaded' in template.status or template.status == 'Processing': retries = retries - 1 continue diff --git a/tools/marvin/mvn-setup.py b/tools/marvin/mvn-setup.py index cabcf0dc659..0eab9495512 100755 --- a/tools/marvin/mvn-setup.py +++ b/tools/marvin/mvn-setup.py @@ -33,7 +33,7 @@ def replaceVersion(fname, version): """replace VERSION in setup.py""" with open(fname, 'r') as f: content = f.read() - needle = '\nVERSION\s*=\s*[\'"][^\'"]*[\'"]' + needle = r'\nVERSION\s*=\s*[\'"][^\'"]*[\'"]' # Ensure the version is PEP440 compliant version = version.replace('-', '+', 1) replacement = '\nVERSION = "%s"' % version diff --git a/tools/marvin/setup.py b/tools/marvin/setup.py index 6c9aa087bc7..4694cecc8a5 100644 --- a/tools/marvin/setup.py +++ b/tools/marvin/setup.py @@ -38,7 +38,7 @@ setup(name="Marvin", maintainer_email="dev@cloudstack.apache.org", long_description="Marvin is the Apache CloudStack python " "client written around the unittest framework", - platforms=("Any",), + platforms=["Any"], url="https://builds.apache.org/job/cloudstack-marvin/", packages=["marvin", "marvin.cloudstackAPI", "marvin.lib", "marvin.config", "marvin.sandbox", diff --git a/tools/utils/cloud-image-downloader.sh b/tools/utils/cloud-image-downloader.sh new file mode 100755 index 00000000000..90f234906e1 --- /dev/null +++ b/tools/utils/cloud-image-downloader.sh @@ -0,0 +1,259 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +#------------------------------------------------------------------------------- +# Configuration +#------------------------------------------------------------------------------- +# This section contains the variables you might want to change. + +# The temporary directory where files will be downloaded. +# It's a good practice to create a unique temporary directory for each script run. +TEMP_DIR=$(mktemp -d) + +# The BASE destination directory for the downloaded image files. +# Subdirectories for each distro will be created inside this one. +# Make sure this directory exists before running the script. +# Must be executed by the cloudstack user on machine hosting the public download site. +# It will be publicly available at https://download.cloudstack.org/templates/cloud-images/ +DEST_DIR="${HOME}/repository/templates/cloud-images" + +# The directory where log files will be stored. +# Make sure this directory exists. +LOG_DIR="${HOME}/log/cloud-image-downloader" +LOG_FILE="${LOG_DIR}/cloud-image-downloader_$(date +%Y%m%d_%H%M%S).log" +LOG_RETENTION_DAYS=30 + +LOGGER_TAG="cloud-image-downloader" +LOGGER_FACILITY="user" +LOGGER_AVAILABLE=false + +log_message() { + local priority=$1 + shift + local message="$*" + local timestamp=$(date +'%Y-%m-%d %H:%M:%S') + + # Log to file + echo "${timestamp} [${priority}] ${message}" | tee -a "${LOG_FILE}" + + # Log to syslog using logger utility + if [ "${LOGGER_AVAILABLE}" = true ]; then + logger -t "${LOGGER_TAG}" -p "${LOGGER_FACILITY}.${priority}" -- "${message}" + fi +} + +log_info() { + log_message "info" "$@" +} + +log_warn() { + log_message "warning" "$@" +} + +log_error() { + log_message "err" "$@" +} + +cleanup_old_logs() { + log_info "Cleaning up log files older than ${LOG_RETENTION_DAYS} days..." + + if [ ! -d "$LOG_DIR" ]; then + log_warn "Log directory does not exist: $LOG_DIR" + return + fi + + local deleted_count=0 + + # Find and delete log files older than retention period + while IFS= read -r -d '' log_file; do + rm -f "$log_file" + deleted_count=$((deleted_count + 1)) + done < <(find "$LOG_DIR" -name "*.log" -type f -mtime +${LOG_RETENTION_DAYS} -print0 2>/dev/null) + + if [ $deleted_count -gt 0 ]; then + log_info "Deleted $deleted_count old log file(s)" + else + log_info "No old log files to delete" + fi +} + +#------------------------------------------------------------------------------- +# Image Definitions +#------------------------------------------------------------------------------- +# To add a new image, you must add an entry to BOTH arrays below. + +# 1. Add the destination filename and the download URL. +declare -A IMAGE_URLS=( + ["Rocky-9-GenericCloud.latest.x86_64.qcow2"]="https://dl.rockylinux.org/pub/rocky/9/images/x86_64/Rocky-9-GenericCloud.latest.x86_64.qcow2" + ["Rocky-9-GenericCloud.latest.aarch64.qcow2"]="https://dl.rockylinux.org/pub/rocky/9/images/aarch64/Rocky-9-GenericCloud.latest.aarch64.qcow2" + ["Rocky-8-GenericCloud.latest.x86_64.qcow2"]="https://dl.rockylinux.org/pub/rocky/8/images/x86_64/Rocky-8-GenericCloud.latest.x86_64.qcow2" + ["Rocky-8-GenericCloud.latest.aarch64.qcow2"]="https://dl.rockylinux.org/pub/rocky/8/images/aarch64/Rocky-8-GenericCloud.latest.aarch64.qcow2" + ["openSUSE-Leap-15.5-Minimal-VM.x86_64-Cloud.qcow2"]="https://download.opensuse.org/distribution/leap/15.5/appliances/openSUSE-Leap-15.5-Minimal-VM.x86_64-Cloud.qcow2" + ["openSUSE-Leap-15.5-Minimal-VM.aarch64-Cloud.qcow2"]="https://download.opensuse.org/distribution/leap/15.5/appliances/openSUSE-Leap-15.5-Minimal-VM.aarch64-Cloud.qcow2" + ["debian-12-genericcloud-amd64.qcow2"]="https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-genericcloud-amd64.qcow2" + ["debian-12-genericcloud-arm64.qcow2"]="https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-genericcloud-arm64.qcow2" + ["ubuntu-24.04-server-cloudimg-amd64.img"]="https://cloud-images.ubuntu.com/releases/24.04/release/ubuntu-24.04-server-cloudimg-amd64.img" + ["ubuntu-24.04-server-cloudimg-arm64.img"]="https://cloud-images.ubuntu.com/releases/24.04/release/ubuntu-24.04-server-cloudimg-arm64.img" + ["ubuntu-22.04-server-cloudimg-amd64.img"]="https://cloud-images.ubuntu.com/releases/22.04/release/ubuntu-22.04-server-cloudimg-amd64.img" + ["ubuntu-22.04-server-cloudimg-arm64.img"]="https://cloud-images.ubuntu.com/releases/22.04/release/ubuntu-22.04-server-cloudimg-arm64.img" + ["ubuntu-20.04-server-cloudimg-amd64.img"]="https://cloud-images.ubuntu.com/releases/20.04/release/ubuntu-20.04-server-cloudimg-amd64.img" + ["ubuntu-20.04-server-cloudimg-arm64.img"]="https://cloud-images.ubuntu.com/releases/20.04/release/ubuntu-20.04-server-cloudimg-arm64.img" + ["OL9U5_x86_64-kvm-b259.qcow2"]="https://yum.oracle.com/templates/OracleLinux/OL9/u5/x86_64/OL9U5_x86_64-kvm-b259.qcow2" + ["OL9U5_aarch64-kvm-b126.qcow2"]="https://yum.oracle.com/templates/OracleLinux/OL9/u5/aarch64/OL9U5_aarch64-kvm-b126.qcow2" + ["OL8U10_x86_64-kvm-b258.qcow2"]="https://yum.oracle.com/templates/OracleLinux/OL8/u10/x86_64/OL8U10_x86_64-kvm-b258.qcow2" + ["OL8U10_aarch64-kvm-b122.qcow2"]="https://yum.oracle.com/templates/OracleLinux/OL8/u10/aarch64/OL8U10_aarch64-kvm-b122.qcow2" +) + +# 2. Add the destination filename and its corresponding distribution subdirectory name. +declare -A IMAGE_DISTROS=( + ["Rocky-9-GenericCloud.latest.x86_64.qcow2"]="rockylinux" + ["Rocky-9-GenericCloud.latest.aarch64.qcow2"]="rockylinux" + ["Rocky-8-GenericCloud.latest.x86_64.qcow2"]="rockylinux" + ["Rocky-8-GenericCloud.latest.aarch64.qcow2"]="rockylinux" + ["openSUSE-Leap-15.5-Minimal-VM.x86_64-Cloud.qcow2"]="opensuse" + ["openSUSE-Leap-15.5-Minimal-VM.aarch64-Cloud.qcow2"]="opensuse" + ["debian-12-genericcloud-amd64.qcow2"]="debian" + ["debian-12-genericcloud-arm64.qcow2"]="debian" + ["ubuntu-24.04-server-cloudimg-amd64.img"]="ubuntu" + ["ubuntu-24.04-server-cloudimg-arm64.img"]="ubuntu" + ["ubuntu-22.04-server-cloudimg-amd64.img"]="ubuntu" + ["ubuntu-22.04-server-cloudimg-arm64.img"]="ubuntu" + ["ubuntu-20.04-server-cloudimg-amd64.img"]="ubuntu" + ["ubuntu-20.04-server-cloudimg-arm64.img"]="ubuntu" + ["OL9U5_x86_64-kvm-b259.qcow2"]="oraclelinux" + ["OL9U5_aarch64-kvm-b126.qcow2"]="oraclelinux" + ["OL8U10_x86_64-kvm-b258.qcow2"]="oraclelinux" + ["OL8U10_aarch64-kvm-b122.qcow2"]="oraclelinux" +) + +#------------------------------------------------------------------------------- +# Cleanup Handler +#------------------------------------------------------------------------------- + +cleanup_on_exit() { + local exit_code=$? + if [ -d "$TEMP_DIR" ]; then + rm -rf "$TEMP_DIR" + log_info "Temporary directory $TEMP_DIR removed." + fi + + if [ $exit_code -ne 0 ]; then + log_error "Script exited with error code: $exit_code" + fi +} + +trap cleanup_on_exit EXIT INT TERM + +#------------------------------------------------------------------------------- +# Main Script Logic +#------------------------------------------------------------------------------- + +if command -v logger &> /dev/null; then + LOGGER_AVAILABLE=true +fi + +# Ensure base destination and log directories exist +mkdir -p "$DEST_DIR" +mkdir -p "$LOG_DIR" + +# Clean up old logs first +cleanup_old_logs + +log_info "Starting image download process." +log_info "Temporary directory: $TEMP_DIR" +log_info "Base destination directory: $DEST_DIR" +log_info "Log file: $LOG_FILE" + +# Inform about logger status +if [ "${LOGGER_AVAILABLE}" = true ]; then + log_info "Syslog logging enabled (tag: ${LOGGER_TAG})" +else + log_warn "Syslog logging disabled - logger utility not found" +fi + +# Loop through the image URLs +for filename in "${!IMAGE_URLS[@]}"; do + url="${IMAGE_URLS[$filename]}" + distro="${IMAGE_DISTROS[$filename]}" + + # Check if a distro is defined for the file + if [ -z "$distro" ]; then + log_error "No distribution directory defined for $filename. Skipping." + continue + fi + + distro_dest_dir="${DEST_DIR}/${distro}" + temp_filepath="${TEMP_DIR}/${filename}" + dest_filepath="${distro_dest_dir}/${filename}" + + log_info "--------------------------------------------------" + log_info "Starting download for: $filename" + log_info "URL: $url" + + # Download the file to the temporary directory + wget --progress=bar:force:noscroll -O "$temp_filepath" "$url" + download_status=$? + + if [ $download_status -ne 0 ]; then + # Handle download failure + log_error "Failed to download $filename from $url. wget exit code: $download_status" + else + # Handle download success + log_info "Successfully downloaded $filename to temporary location." + + # Ensure the specific distro directory exists + log_info "Ensuring destination directory exists: $distro_dest_dir" + mkdir -p "$distro_dest_dir" + + # Move the file to the destination directory, replacing any existing file + log_info "Moving $filename to $dest_filepath" + mv -f "$temp_filepath" "$dest_filepath" + move_status=$? + + if [ $move_status -ne 0 ]; then + log_error "Failed to move $filename to $dest_filepath. mv exit code: $move_status" + else + log_info "Successfully moved $filename." + fi + fi +done + +log_info "Generate checksum" +# Create md5 checksum +checksum_file="md5sum.txt" +sha512_checksum_file="sha512sum.txt" + +cd "$DEST_DIR" +find . -type f ! -iname '*.txt' -exec md5sum {} \; > "$checksum_file" +checksum_status=$? +if [ $checksum_status -ne 0 ]; then + log_error "Failed to create md5 checksum. md5sum exit code: $checksum_status" +else + log_info "Successfully created checksum file: $checksum_file" +fi + +find . -type f ! -iname '*.txt' -exec sha512sum {} \; > "$sha512_checksum_file" +sha512_checksum_status=$? +if [ $sha512_checksum_status -ne 0 ]; then + log_error "Failed to create sha512 checksum. sha512sum exit code: $sha512_checksum_status" +else + log_info "Successfully created checksum file: $sha512_checksum_file" +fi + +log_info "--------------------------------------------------" +log_info "Image download process finished." diff --git a/ui/README.md b/ui/README.md index 170232b574e..3f7bcb8120e 100644 --- a/ui/README.md +++ b/ui/README.md @@ -27,18 +27,18 @@ A modern role-based progressive CloudStack UI based on Vue.js and Ant Design. Install node: (Debian/Ubuntu) - curl -sL https://deb.nodesource.com/setup_20.x | sudo -E bash - + curl -sL https://deb.nodesource.com/setup_24.x | sudo -E bash - sudo apt-get install -y nodejs # Or use distro provided: sudo apt-get install npm nodejs Install node: (CentOS/Fedora/RHEL) - curl -sL https://rpm.nodesource.com/setup_20.x | sudo bash - + curl -sL https://rpm.nodesource.com/setup_24.x | sudo bash - sudo yum install nodejs Install node: (Mac OS) - brew install node@20 + brew install node@24 Optionally, you may also install system-wide dev tools: diff --git a/ui/package.json b/ui/package.json index 48f337500bd..9801c1b1815 100644 --- a/ui/package.json +++ b/ui/package.json @@ -101,15 +101,18 @@ "eslint-plugin-vue": "^7.0.0", "less": "^3.0.4", "less-loader": "^5.0.0", + "nan": "2.18.0", + "node-gyp": "10.0.1", "sass": "^1.49.9", "sass-loader": "^8.0.2", "uglifyjs-webpack-plugin": "^2.2.0", "vue-jest": "^5.0.0-0", "vue-svg-loader": "^0.17.0-beta.2", - "webpack": "^4.46.0", - "node-gyp": "10.0.1", "nan": "2.18.0" + "webpack": "^4.46.0" + }, + "resolutions": { + "nan": "2.18.0" }, - "resolutions": { "nan": "2.18.0" }, "eslintConfig": { "root": true, "env": { diff --git a/ui/public/config.json b/ui/public/config.json index 1a7beda654e..de3a3c39952 100644 --- a/ui/public/config.json +++ b/ui/public/config.json @@ -105,6 +105,7 @@ "showAllCategoryForModernImageSelection": false, "docHelpMappings": {}, "notifyLatestCSVersion": true, + "showSearchFilters": true, "announcementBanner": { "enabled": false, "showIcon": false, diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index 21fe35ea157..938ecb3508a 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -42,11 +42,14 @@ "label.accounts": "Accounts", "label.accountstate": "Account state", "label.accounttype": "Account type", -"label.acl.export": "Export ACL rules", +"label.import": "Import", +"label.acl.import": "Import rules", +"label.acl.export": "Export rules", "label.acl.id": "ACL ID", "label.acl.rules": "ACL rules", "label.acl.reason.description": "Enter the reason behind an ACL rule.", "label.aclid": "ACL", +"label.aclname": "ACL name", "label.acl.rule.name": "ACL rule name", "label.acquire.new.ip": "Acquire new IP", "label.acquire.new.secondary.ip": "Acquire new secondary IP", @@ -254,7 +257,7 @@ "label.add": "Add", "label.addservices": "Add Services", "label.add.account": "Add Account", -"label.add.acl.rule": "Add ACL rule", +"label.add.acl.rule": "Add rule", "label.add.acl": "Add ACL", "label.add.affinity.group": "Add new Affinity Group", "label.add.backup.schedule": "Add Backup Schedule", @@ -292,8 +295,10 @@ "label.add.isolated.network": "Add Isolated Network", "label.add.kubernetes.cluster": "Add Kubernetes Cluster", "label.add.acl.name": "ACL name", +"label.add.latest.kubernetes.iso": "Add latest Kubernetes ISO", "label.add.ldap.account": "Add LDAP Account", "label.add.logical.router": "Add Logical Router to this Network", +"label.add.minimum.required.compute.offering": "Add minimum required Compute Offering", "label.add.more": "Add more", "label.add.nodes": "Add Nodes to Kubernetes Cluster", "label.add.netscaler.device": "Add Netscaler Device", @@ -440,6 +445,8 @@ "label.auto.assign": "Automatically assign", "label.auto.assign.diskoffering.disk.size": "Automatically assign offering matching the disk size", "label.auto.assign.random.ip": "Automatically assign a random IP address", +"label.auto.refresh.statistics": "Period between auto refreshes", +"label.auto.refresh.statistics.none": "None", "label.automigrate.volume": "Auto migrate volume to another storage pool if required", "label.autoscale.vm.groups": "AutoScaling Groups", "label.autoscale.vm.profile": "AutoScale Instance Profile", @@ -655,6 +662,8 @@ "label.copy.consoleurl": "Copy console URL to clipboard", "label.copyid": "Copy ID", "label.copy.password": "Copy password", +"label.copy.templates.from.other.secondary.storages": "Copy Templates from other storages instead of fetching from URLs", +"label.copy.templates.from.other.secondary.storages.add.zone": "Copy Templates from other storages", "label.core": "Core", "label.core.zone.type": "Core Zone type", "label.count": "Count", @@ -712,6 +721,7 @@ "label.cron.mode": "Cron mode", "label.crosszones": "Cross Zones", "label.csienabled": "CSI Enabled", +"label.csv.preview": "Data preview", "label.currency": "Currency", "label.current": "Current", "label.currentstep": "Current step", @@ -1107,6 +1117,7 @@ "label.firstname": "First name", "label.firstname.lower": "firstname", "label.fix.errors": "Fix errors", +"label.fix.global.setting": "Fix Global Setting", "label.fixed": "Fixed Offering", "label.for": "for", "label.forcks": "For CKS", @@ -1140,6 +1151,9 @@ "label.globo.dns.configuration": "GloboDNS configuration", "label.glustervolume": "Volume", "label.go.back": "Go back", +"label.go.to.compute.offerings": "Go to Compute Offerings", +"label.go.to.global.settings": "Go to Global Settings", +"label.go.to.kubernetes.isos": "Go to Kubernetes ISOs", "label.gpu": "GPU", "label.gpucardid": "GPU Card", "label.gpucardname": "GPU Card", @@ -1390,6 +1404,7 @@ "label.isoname": "Attached ISO", "label.isos": "ISOs", "label.isostate": "ISO state", +"label.isourl": "ISO URL", "label.ispersistent": "Persistent ", "label.ispublic": "Public", "label.isready": "Ready", @@ -2445,6 +2460,7 @@ "label.storagepool.tooltip": "Destination Storage Pool. Volume should be located in this Storage Pool", "label.storagetags": "Storage tags", "label.storagetype": "Storage type", +"label.storageip": "Storage IP address", "label.strict": "Strict", "label.subdomainaccess": "Subdomain access", "label.submit": "Submit", @@ -2773,8 +2789,8 @@ "label.vnf.app.action.reinstall": "Reinstall VNF Appliance", "label.vnf.cidr.list": "CIDR from which access to the VNF appliance's Management interface should be allowed from", "label.vnf.cidr.list.tooltip": "the CIDR list to forward traffic from to the VNF management interface. Multiple entries must be separated by a single comma character (,). The default value is 0.0.0.0/0.", -"label.vnf.configure.management": "Configure Firewall and Port Forwarding rules for VNF's management interfaces", -"label.vnf.configure.management.tooltip": "True by default, security group or network rules (source nat and firewall rules) will be configured for VNF management interfaces. False otherwise. Learn what rules are configured at http://docs.cloudstack.apache.org/en/latest/adminguide/networking/vnf_templates_appliances.html#deploying-vnf-appliances", +"label.vnf.configure.management": "Configure network rules for VNF's management interfaces", +"label.vnf.configure.management.tooltip": "False by default, security group or network rules (source nat and firewall rules) will be configured for VNF management interfaces. True otherwise. Learn what rules are configured at http://docs.cloudstack.apache.org/en/latest/adminguide/networking/vnf_templates_appliances.html#deploying-vnf-appliances", "label.vnf.detail.add": "Add VNF detail", "label.vnf.detail.remove": "Remove VNF detail", "label.vnf.details": "VNF Details", @@ -2920,6 +2936,8 @@ "message.action.create.snapshot.from.vmsnapshot": "Please confirm that you want to create Snapshot from Instance Snapshot", "message.action.create.instance.from.backup": "Please confirm that you want to create a new Instance from the given Backup.
    Click on configure to edit the parameters for the new Instance before creation.", "message.create.instance.from.backup.different.zone": "Creating Instance from Backup on a different Zone. Please ensure that the backup repository is accessible in the selected Zone.", +"message.csv.empty": "Empty CSV File", +"message.csv.missing.headers": "Columns are missing from headers in CSV", "message.template.ostype.different.from.backup": "Selected Template has a different OS type than the Backup. Please proceed with caution.", "message.iso.ostype.different.from.backup": "Selected ISO has a different OS type than the Backup. Please proceed with caution.", "message.action.delete.asnrange": "Please confirm the AS range that you want to delete", @@ -3061,12 +3079,17 @@ "message.add.ip.v6.firewall.rule.failed": "Failed to add IPv6 firewall rule", "message.add.ip.v6.firewall.rule.processing": "Adding IPv6 firewall rule...", "message.add.ip.v6.firewall.rule.success": "Added IPv6 firewall rule", +"message.advisory.cks.endpoint.url.not.configured": "Endpoint URL which will be used by Kubernetes clusters is not configured correctly", +"message.advisory.cks.min.offering": "No suitable Compute Offering found for Kubernetes cluster nodes with minimum required resources (2 vCPU, 2 GB RAM)", +"message.advisory.cks.version.check": "No Kubernetes version found that can be used to deploy a Kubernetes cluster", "message.redeliver.webhook.delivery": "Redeliver this Webhook delivery", "message.remove.ip.v6.firewall.rule.failed": "Failed to remove IPv6 firewall rule", "message.remove.ip.v6.firewall.rule.processing": "Removing IPv6 firewall rule...", "message.remove.ip.v6.firewall.rule.success": "Removed IPv6 firewall rule", "message.remove.sslcert.failed": "Failed to remove SSL certificate from load balancer", "message.remove.sslcert.processing": "Removing SSL certificate from load balancer...", +"message.add.latest.kubernetes.iso.failed": "Failed to add latest Kubernetes ISO", +"message.add.minimum.required.compute.offering.kubernetes.cluster.failed": "Failed to add minimum required Compute Offering for Kubernetes cluster nodes", "message.add.netris.controller": "Add Netris Provider", "message.add.nsx.controller": "Add NSX Provider", "message.add.network": "Add a new network for Zone: ", @@ -3107,9 +3130,13 @@ "message.add.vpn.gateway": "Please confirm that you want to add a VPN Gateway.", "message.add.vpn.gateway.failed": "Adding VPN gateway failed", "message.add.vpn.gateway.processing": "Adding VPN gateway...", +"message.added.latest.kubernetes.iso": "Latest Kubernetes ISO added successfully", +"message.added.minimum.required.compute.offering.kubernetes.cluster": "Minimum required Compute Offering for Kubernetes cluster nodes added successfully", "message.added.vpc.offering": "Added VPC offering", "message.adding.firewall.policy": "Adding Firewall Policy", "message.adding.host": "Adding host", +"message.adding.latest.kubernetes.iso": "Adding latest Kubernetes ISO", +"message.adding.minimum.required.compute.offering.kubernetes.cluster": "Adding minimum required Compute Offering for Kubernetes cluster nodes", "message.adding.netscaler.device": "Adding Netscaler device", "message.adding.netscaler.provider": "Adding Netscaler provider", "message.adding.nodes.to.cluster": "Adding nodes to Kubernetes cluster", @@ -3338,7 +3365,7 @@ "message.desc.register.user.data": "Please fill in the following to register new User Data.", "message.desc.registered.user.data": "Registered a User Data.", "message.desc.reset.ssh.key.pair": "Please specify a ssh key pair that you would like to add to this Instance.", -"message.desc.secondary.storage": "Each Zone must have at least one NFS or secondary storage server. We will add the first one now. Secondary storage stores Instance Templates, ISO images, and Instance disk volume Snapshots. This server must be available to all hosts in the zone.

    Provide the IP address and exported path.", +"message.desc.secondary.storage": "Each Zone must have at least one NFS or secondary storage server. We will add the first one now. Secondary storage stores Instance Templates, ISO images, and Instance disk volume Snapshots. This server must be available to all hosts in the zone.

    Provide the IP address and exported path.

    \"Copy templates from other secondary storages\" switch can be used to automatically copy existing templates from secondary storages in other zones instead of fetching from their URLs.", "message.desc.validationformat": "Specifies the format used to validate the parameter value, such as EMAIL, URL, UUID, DECIMAL, etc.", "message.desc.valueoptions": "Provide a comma-separated list of values that will appear as selectable options for this parameter", "message.desc.zone": "A Zone is the largest organizational unit in CloudStack, and it typically corresponds to a single datacenter. Zones provide physical isolation and redundancy. A zone consists of one or more Pods (each of which contains hosts and primary storage servers) and a secondary storage server which is shared by all pods in the zone.", @@ -3554,6 +3581,8 @@ "message.failed.to.remove": "Failed to remove", "message.forgot.password.success": "An email has been sent to your email address with instructions on how to reset your password.", "message.generate.keys": "Please confirm that you would like to generate new API/Secret keys for this User.", +"message.global.setting.updated": "Global Setting updated successfully.", +"message.global.setting.update.failed": "Failed to update Global Setting.", "message.chart.statistic.info": "The shown charts are self-adjustable, that means, if the value gets close to the limit or overpass it, it will grow to adjust the shown value", "message.chart.statistic.info.hypervisor.additionals": "The metrics data depend on the hypervisor plugin used for each hypervisor. The behavior can vary across different hypervisors. For instance, with KVM, metrics are real-time statistics provided by libvirt. In contrast, with VMware, the metrics are averaged data for a given time interval controlled by configuration.", "message.guest.traffic.in.advanced.zone": "Guest Network traffic is communication between end-user Instances. Specify a range of VLAN IDs or VXLAN Network identifiers (VNIs) to carry guest traffic for each physical Network.", @@ -3669,6 +3698,7 @@ "message.move.acl.order.processing": "Moving ACL rule...", "message.network.acl.default.allow": "Warning: With this policy all traffic will be allowed through the firewall to this VPC Network Tier. You should consider securing your Network.", "message.network.acl.default.deny": "Warning: With this policy all traffic will be denied through the firewall to this VPC Network Tier. In order to allow traffic through you will need to change policies.", +"message.network.acl.import.note": "Note: Only valid rules from the CSV will be imported. Invalid entries will be discarded.", "message.network.addvm.desc": "Please specify the Network that you would like to add this Instance to. A new NIC will be added for this Network.", "message.network.description": "Setup Network and traffic.", "message.network.error": "Network Error", @@ -3761,6 +3791,7 @@ "message.resource.not.found": "Resource not found.", "message.restart.mgmt.server": "Please restart your management server(s) for your new settings to take effect.", "message.restart.network": "All services provided by this Network will be interrupted. Please confirm that you want to restart this Network.", +"message.restart.usage.server": "Please restart your usage server(s) for your new settings to take effect.", "message.restart.vm.to.update.settings": "Update in fields other than name and display name will require the Instance to be restarted.", "message.restart.vpc": "Please confirm that you want to restart the VPC.", "message.restart.vpc.remark": "Please confirm that you want to restart the VPC

    Remark: making a non-redundant VPC redundant will force a clean up. The Networks will not be available for a couple of minutes.

    ", diff --git a/ui/public/locales/pt_BR.json b/ui/public/locales/pt_BR.json index 4346f1f0bb7..59123a5e45c 100644 --- a/ui/public/locales/pt_BR.json +++ b/ui/public/locales/pt_BR.json @@ -288,6 +288,8 @@ "label.author.name": "Nome do autor", "label.auto.assign.diskoffering.disk.size": "Atribuir automaticamente a oferta correspondente ao tamanho do disco", "label.auto.assign.random.ip": "Atribuir automaticamente um enderec\u0327o de IP aleat\u00f3rio", +"label.auto.refresh.statistics": "Tempo entre atualiza\u00e7\u00f5es autom\u00e1ticas", +"label.auto.refresh.statistics.none": "Nenhum", "label.autoscalingenabled": "Escalonamento autom\u00e1tico", "label.availability": "Disponibilidade", "label.available": "Dispon\u00edvel", @@ -610,6 +612,12 @@ "label.download.state": "Estado do download", "label.dpd": "Detec\u00e7\u00e3o de correspondente morto", "label.driver": "Driver", +"label.duration.custom": "Personalizado", +"label.duration.1hour": "1 hora", +"label.duration.6hours": "6 horas", +"label.duration.12hours": "12 horas", +"label.duration.24hours": "24 horas", +"label.duration.7days": "7 dias", "label.dynamicscalingenabled": "Escalonamento din\u00e2mico habilitado", "label.dynamicscalingenabled.tooltip": "VM s\u00f3 pode ser dinamicamente escalonada quando o escalonamento din\u00e2mico estiver habilitado no template, oferta de computa\u00e7\u00e3o e nas configura\u00e7\u00e3oes globais", "label.edit": "Editar", @@ -881,6 +889,7 @@ "label.isoname": "Imagem ISO plugada", "label.isos": "ISOs", "label.isostate": "Estado da ISO", +"label.isourl": "URL da ISO", "label.ispersistent": "Persistente", "label.ispublic": "P\u00fablico", "label.isready": "Pronto", @@ -1583,6 +1592,7 @@ "label.storagepool": "Pool de armazenamento", "label.storagetags": "Tags de armazenamento", "label.storagetype": "Tipo de armazenamento", +"label.storageip": "Endere\u00e7o IP na rede de armazenamento", "label.strict": "Rigoroso", "label.subdomainaccess": "acesso ao subdom\u00ednio", "label.submit": "Enviar", diff --git a/ui/src/api/index.js b/ui/src/api/index.js index 0c8e8e9696c..5ec73a0b20e 100644 --- a/ui/src/api/index.js +++ b/ui/src/api/index.js @@ -140,3 +140,7 @@ export function oauthlogin (arg) { } }) } + +export function getBaseUrl () { + return vueProps.axios.defaults.baseURL +} diff --git a/ui/src/components/view/AdvisoriesView.vue b/ui/src/components/view/AdvisoriesView.vue new file mode 100644 index 00000000000..c668ec02380 --- /dev/null +++ b/ui/src/components/view/AdvisoriesView.vue @@ -0,0 +1,161 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + + + diff --git a/ui/src/components/view/DedicateDomain.vue b/ui/src/components/view/DedicateDomain.vue index 01224efc0c3..a1d69dcd843 100644 --- a/ui/src/components/view/DedicateDomain.vue +++ b/ui/src/components/view/DedicateDomain.vue @@ -18,52 +18,44 @@ diff --git a/ui/src/components/view/SettingsTab.vue b/ui/src/components/view/SettingsTab.vue index ab75bef8394..0e0ee33f280 100644 --- a/ui/src/components/view/SettingsTab.vue +++ b/ui/src/components/view/SettingsTab.vue @@ -26,7 +26,8 @@ + :resource="resource" + @refresh-config="handleConfigRefresh" /> @@ -140,6 +141,13 @@ export default { handleSearch (value) { this.filter = value this.fetchData() + }, + handleConfigRefresh (name, updatedRecord) { + if (!name || !updatedRecord) return + const index = this.items.findIndex(item => item.name === name) + if (index !== -1) { + this.items.splice(index, 1, updatedRecord) + } } } } diff --git a/ui/src/components/view/StatsTab.vue b/ui/src/components/view/StatsTab.vue index 8703b6b995d..d8739ece6c1 100644 --- a/ui/src/components/view/StatsTab.vue +++ b/ui/src/components/view/StatsTab.vue @@ -41,9 +41,9 @@ + @change="updateVirtualMachineStats"> - {{ $t('1 hour') }} + {{ $t('label.duration.1hour') }} {{ $t('label.duration.6hours') }} @@ -62,6 +62,16 @@ + {{$t('label.auto.refresh.statistics')}} + + {{$t('label.auto.refresh.statistics.none')}} + 5s + 30s + 1min + 5min +
    @@ -297,6 +307,8 @@ export default { selectedDiskUnitOfMeasurement: 'KiB', diskUnitsOfMeasurement: ['KiB', 'MiB', 'GiB'], chartLabels: [], + refreshTime: '0', + refreshIntervalId: null, resourceUsageHistory: { cpu: [], memory: { @@ -334,6 +346,9 @@ export default { mounted () { this.fetchData() }, + unmounted () { + window.clearInterval(this.refreshIntervalId) + }, computed: { statsRetentionTime () { if (this.resourceType === 'Volume') { @@ -371,6 +386,15 @@ export default { return } this.fetchData() + }, + refreshTime: function () { + this.updateVirtualMachineStats() + if (this.refreshTime === '0') return window.clearInterval(this.refreshIntervalId) + + window.clearInterval(this.refreshIntervalId) + this.refreshIntervalId = window.setInterval(() => { + this.updateVirtualMachineStats() + }, parseInt(this.refreshTime)) } }, methods: { @@ -398,26 +422,10 @@ export default { this.resourceTypeToShowInfo = resource this.showResourceInfoModal = true }, - handleDurationChange () { - var now = this.getEndDate() - var start = new Date(now) - switch (this.durationSelectorValue) { - case '6hours': - start.setHours(start.getHours() - 6) - break - case '12hours': - start.setHours(start.getHours() - 12) - break - case 'day': - start.setDate(start.getDate() - 1) - break - case 'week': - start.setDate(start.getDate() - 7) - break - default: - start.setHours(start.getHours() - 1) - } - this.handleSubmit({ startDate: start, endDate: now }) + updateVirtualMachineStats () { + const start = this.getStartDate() + const end = this.getEndDate() + this.handleSubmit({ startDate: start, endDate: end }) }, handleSubmit (values) { if (values.startDate) { @@ -437,9 +445,19 @@ export default { this.showFilterStatsModal = false }, getStartDate () { - var now = new Date() - now.setHours(now.getHours() - 1) - return now + const now = new Date() + switch (this.durationSelectorValue) { + case '6hours': + return now.setHours(now.getHours() - 6) + case '12hours': + return now.setHours(now.getHours() - 12) + case 'day': + return now.setDate(now.getDate() - 1) + case 'week': + return now.setDate(now.getDate() - 7) + default: + return now.setHours(now.getHours() - 1) + } }, getEndDate () { return new Date() diff --git a/ui/src/components/widgets/InfiniteScrollSelect.vue b/ui/src/components/widgets/InfiniteScrollSelect.vue index 122feafb2a0..8db4d8d523a 100644 --- a/ui/src/components/widgets/InfiniteScrollSelect.vue +++ b/ui/src/components/widgets/InfiniteScrollSelect.vue @@ -41,9 +41,11 @@ - optionValueKey (String, optional): Property to use as the value for options (e.g., 'name'). Default is 'id' - optionLabelKey (String, optional): Property to use as the label for options (e.g., 'name'). Default is 'name' - defaultOption (Object, optional): Preselected object to include initially + - allowClear (Boolean, optional): Whether to allow clearing the selection. Default is false - showIcon (Boolean, optional): Whether to show icon for the options. Default is true - defaultIcon (String, optional): Icon to be shown when there is no resource icon for the option. Default is 'cloud-outlined' - autoSelectFirstOption (Boolean, optional): Whether to automatically select the first option when options are loaded. Default is false + - selectFirstOption (Boolean, optional): Whether to automatically select the first option when options are loaded. Default is false Events: - @change-option-value (Function): Emits the selected option value(s) when value(s) changes. Do not use @change as it will give warnings and may not work @@ -59,6 +61,7 @@ :filter-option="false" :loading="loading" show-search + :allowClear="allowClear" placeholder="Select" @search="onSearchTimed" @popupScroll="onScroll" @@ -76,9 +79,9 @@
    - + - + @@ -129,6 +132,10 @@ export default { type: Object, default: null }, + allowClear: { + type: Boolean, + default: false + }, showIcon: { type: Boolean, default: true @@ -144,6 +151,10 @@ export default { autoSelectFirstOption: { type: Boolean, default: false + }, + selectFirstOption: { + type: Boolean, + default: false } }, data () { @@ -157,7 +168,8 @@ export default { scrollHandlerAttached: false, preselectedOptionValue: null, successiveFetches: 0, - canSelectFirstOption: false + canSelectFirstOption: false, + hasAutoSelectedFirst: false } }, created () { @@ -176,6 +188,36 @@ export default { }, formattedSearchFooterMessage () { return `${this.$t('label.showing.results.for').replace('%x', this.searchQuery)}` + }, + selectableOptions () { + const currentValue = this.$attrs.value + // Only filter out null/empty options when the current value is also null/undefined/empty + // This prevents such options from being selected and allows the placeholder to show instead + if (currentValue === null || currentValue === undefined || currentValue === '') { + return this.options.filter(option => { + const optionValue = option[this.optionValueKey] + return optionValue !== null && optionValue !== undefined && optionValue !== '' + }) + } + // When a valid value is selected, show all options + return this.options + }, + apiOptionsCount () { + if (this.defaultOption) { + const defaultOptionValue = this.defaultOption[this.optionValueKey] + return this.options.filter(option => option[this.optionValueKey] !== defaultOptionValue).length + } + return this.options.length + }, + preselectedMatchValue () { + // Extract the first value from preselectedOptionValue if it's an array, otherwise return the value itself + if (!this.preselectedOptionValue) return null + return Array.isArray(this.preselectedOptionValue) ? this.preselectedOptionValue[0] : this.preselectedOptionValue + }, + preselectedMatch () { + // Find the matching option for the preselected value + if (!this.preselectedMatchValue) return null + return this.options.find(entry => entry[this.optionValueKey] === this.preselectedMatchValue) || null } }, watch: { @@ -221,6 +263,7 @@ export default { this.canSelectFirstOption = true if (this.successiveFetches === 0) { this.loading = false + this.autoSelectFirstOptionIfNeeded() } }) }, @@ -237,11 +280,10 @@ export default { } return } - const matchValue = Array.isArray(this.preselectedOptionValue) ? this.preselectedOptionValue[0] : this.preselectedOptionValue - const match = this.options.find(entry => entry[this.optionValueKey] === matchValue) - if (!match) { + if (!this.preselectedMatch) { this.successiveFetches++ - if (this.options.length < this.totalCount) { + // Exclude defaultOption from count when comparing with totalCount + if (this.apiOptionsCount < this.totalCount) { this.fetchItems() } else { this.resetPreselectedOptionValue() @@ -249,7 +291,7 @@ export default { return } if (Array.isArray(this.preselectedOptionValue) && this.preselectedOptionValue.length > 1) { - this.preselectedOptionValue = this.preselectedOptionValue.filter(o => o !== match) + this.preselectedOptionValue = this.preselectedOptionValue.filter(o => o !== this.preselectedMatchValue) } else { this.resetPreselectedOptionValue() } @@ -264,6 +306,36 @@ export default { this.preselectedOptionValue = null this.successiveFetches = 0 }, + autoSelectFirstOptionIfNeeded () { + if (!this.selectFirstOption || this.hasAutoSelectedFirst) { + return + } + // Don't auto-select if there's a preselected value being fetched + if (this.preselectedOptionValue) { + return + } + const currentValue = this.$attrs.value + if (currentValue !== undefined && currentValue !== null && currentValue !== '') { + return + } + if (this.options.length === 0) { + return + } + if (this.searchQuery && this.searchQuery.length > 0) { + return + } + // Only auto-select after initial load is complete (no more successive fetches) + if (this.successiveFetches > 0) { + return + } + const firstOption = this.options[0] + if (firstOption) { + const firstValue = firstOption[this.optionValueKey] + this.hasAutoSelectedFirst = true + this.$emit('change-option-value', firstValue) + this.$emit('change-option', firstOption) + } + }, onSearchTimed (value) { clearTimeout(this.searchTimer) this.searchTimer = setTimeout(() => { @@ -282,7 +354,8 @@ export default { }, onScroll (e) { const nearBottom = e.target.scrollTop + e.target.clientHeight >= e.target.scrollHeight - 10 - const hasMore = this.options.length < this.totalCount + // Exclude defaultOption from count when comparing with totalCount + const hasMore = this.apiOptionsCount < this.totalCount if (nearBottom && hasMore && !this.loading) { this.fetchItems() } diff --git a/ui/src/components/widgets/OsLogo.vue b/ui/src/components/widgets/OsLogo.vue index 643953012c1..f19aac56a1a 100644 --- a/ui/src/components/widgets/OsLogo.vue +++ b/ui/src/components/widgets/OsLogo.vue @@ -31,6 +31,9 @@ - - diff --git a/ui/src/config/router.js b/ui/src/config/router.js index 08df799dd89..3e5d8677b34 100644 --- a/ui/src/config/router.js +++ b/ui/src/config/router.js @@ -81,6 +81,7 @@ function generateRouterMap (section) { filters: child.filters, params: child.params ? child.params : {}, columns: child.columns, + advisories: !vueProps.$config.advisoriesDisabled ? child.advisories : undefined, details: child.details, searchFilters: child.searchFilters, related: child.related, @@ -180,6 +181,10 @@ function generateRouterMap (section) { map.meta.columns = section.columns } + if (!vueProps.$config.advisoriesDisabled && section.advisories) { + map.meta.advisories = section.advisories + } + if (section.actions) { map.meta.actions = section.actions } diff --git a/ui/src/config/section/compute.js b/ui/src/config/section/compute.js index a03693e351d..32e888bb53d 100644 --- a/ui/src/config/section/compute.js +++ b/ui/src/config/section/compute.js @@ -18,6 +18,8 @@ import { shallowRef, defineAsyncComponent } from 'vue' import store from '@/store' import { isZoneCreated } from '@/utils/zone' +import { getAPI, postAPI, getBaseUrl } from '@/api' +import { getLatestKubernetesIsoParams } from '@/utils/acsrepo' import kubernetesIcon from '@/assets/icons/kubernetes.svg?inline' export default { @@ -582,6 +584,182 @@ export default { } ], resourceType: 'KubernetesCluster', + advisories: [ + { + id: 'cks-min-offering', + severity: 'warning', + message: 'message.advisory.cks.min.offering', + docsHelp: 'plugins/cloudstack-kubernetes-service.html', + dismissOnConditionFail: true, + condition: async (store) => { + if (!('listServiceOfferings' in store.getters.apis)) { + return false + } + const params = { + cpunumber: 2, + memory: 2048, + issystem: false + } + try { + const json = await getAPI('listServiceOfferings', params) + const offerings = json?.listserviceofferingsresponse?.serviceoffering || [] + return !offerings.some(o => !o.iscustomized) + } catch (error) {} + return false + }, + actions: [ + { + primary: true, + label: 'label.add.minimum.required.compute.offering', + loadingLabel: 'message.adding.minimum.required.compute.offering.kubernetes.cluster', + show: (store) => { return ('createServiceOffering' in store.getters.apis) }, + run: async () => { + const params = { + name: 'CKS Instance', + cpunumber: 2, + cpuspeed: 1000, + memory: 2048, + iscustomized: false, + issystem: false + } + try { + const json = await postAPI('createServiceOffering', params) + if (json?.createserviceofferingresponse?.serviceoffering) { + return true + } + } catch (error) {} + return false + }, + successMessage: 'message.added.minimum.required.compute.offering.kubernetes.cluster', + errorMessage: 'message.add.minimum.required.compute.offering.kubernetes.cluster.failed' + }, + { + label: 'label.go.to.compute.offerings', + show: (store) => { return ('listServiceOfferings' in store.getters.apis) }, + run: (store, router) => { + router.push({ name: 'computeoffering' }) + return false + } + } + ] + }, + { + id: 'cks-version-check', + severity: 'warning', + message: 'message.advisory.cks.version.check', + docsHelp: 'plugins/cloudstack-kubernetes-service.html', + dismissOnConditionFail: true, + condition: async (store) => { + const api = 'listKubernetesSupportedVersions' + if (!(api in store.getters.apis)) { + return false + } + try { + const json = await getAPI(api, {}) + const versions = json?.listkubernetessupportedversionsresponse?.kubernetessupportedversion || [] + return versions.length === 0 + } catch (error) {} + return false + }, + actions: [ + { + primary: true, + label: 'label.add.latest.kubernetes.iso', + loadingLabel: 'message.adding.latest.kubernetes.iso', + show: (store) => { return ('addKubernetesSupportedVersion' in store.getters.apis) }, + run: async () => { + let arch = 'x86_64' + if ('listClusters' in store.getters.apis) { + try { + const json = await getAPI('listClusters', { allocationstate: 'Enabled', page: 1, pagesize: 1 }) + const cluster = json?.listclustersresponse?.cluster?.[0] || {} + arch = cluster.architecture || 'x86_64' + } catch (error) {} + } + const params = await getLatestKubernetesIsoParams(arch) + try { + const json = await postAPI('addKubernetesSupportedVersion', params) + if (json?.addkubernetessupportedversionresponse?.kubernetessupportedversion) { + return true + } + } catch (error) {} + return false + }, + successMessage: 'message.added.latest.kubernetes.iso', + errorMessage: 'message.add.latest.kubernetes.iso.failed' + }, + { + label: 'label.go.to.kubernetes.isos', + show: true, + run: (store, router) => { + router.push({ name: 'kubernetesiso' }) + return false + } + } + ] + }, + { + id: 'cks-endpoint-url', + severity: 'warning', + message: 'message.advisory.cks.endpoint.url.not.configured', + docsHelp: 'plugins/cloudstack-kubernetes-service.html', + dismissOnConditionFail: true, + condition: async (store) => { + if (!['Admin'].includes(store.getters.userInfo.roletype)) { + return false + } + let url = '' + const baseUrl = getBaseUrl() + if (baseUrl.startsWith('/')) { + url = window.location.origin + baseUrl + } + if (!url || url.startsWith('http://localhost')) { + return false + } + const params = { + name: 'endpoint.url' + } + const json = await getAPI('listConfigurations', params) + const configuration = json?.listconfigurationsresponse?.configuration?.[0] || {} + return !configuration.value || configuration.value.startsWith('http://localhost') + }, + actions: [ + { + primary: true, + label: 'label.fix.global.setting', + show: (store) => { return ('updateConfiguration' in store.getters.apis) }, + run: async () => { + let url = '' + const baseUrl = getBaseUrl() + if (baseUrl.startsWith('/')) { + url = window.location.origin + baseUrl + } + const params = { + name: 'endpoint.url', + value: url + } + try { + const json = await postAPI('updateConfiguration', params) + if (json?.updateconfigurationresponse?.configuration) { + return true + } + } catch (error) {} + return false + }, + successMessage: 'message.global.setting.updated', + errorMessage: 'message.global.setting.update.failed' + }, + { + label: 'label.go.to.global.settings', + show: (store) => { return ('listConfigurations' in store.getters.apis) }, + run: (store, router) => { + router.push({ name: 'globalsetting' }) + return false + } + } + ] + } + ], actions: [ { api: 'createKubernetesCluster', diff --git a/ui/src/config/section/image.js b/ui/src/config/section/image.js index 6a1d0aae52e..bae4a40c9a8 100644 --- a/ui/src/config/section/image.js +++ b/ui/src/config/section/image.js @@ -61,9 +61,9 @@ export default { details: () => { var fields = ['name', 'id', 'displaytext', 'checksum', 'hypervisor', 'arch', 'format', 'externalprovisioner', 'ostypename', 'size', 'physicalsize', 'isready', 'passwordenabled', 'crossZones', 'templatetype', 'directdownload', 'deployasis', 'ispublic', 'isfeatured', 'isextractable', 'isdynamicallyscalable', 'crosszones', 'type', - 'account', 'domain', 'created', 'userdatadetails', 'userdatapolicy', 'forcks'] + 'account', 'domain', 'created', 'userdatadetails', 'userdatapolicy', 'url', 'forcks'] if (['Admin'].includes(store.getters.userInfo.roletype)) { - fields.push('templatetag', 'templatetype', 'url') + fields.push('templatetag', 'templatetype') } return fields }, @@ -373,7 +373,7 @@ export default { permission: ['listKubernetesSupportedVersions'], searchFilters: ['zoneid', 'minimumsemanticversion', 'arch'], columns: ['name', 'state', 'semanticversion', 'isostate', 'mincpunumber', 'minmemory', 'arch', 'zonename'], - details: ['name', 'semanticversion', 'supportsautoscaling', 'zoneid', 'zonename', 'isoid', 'isoname', 'isostate', 'arch', 'mincpunumber', 'minmemory', 'supportsha', 'state', 'created'], + details: ['name', 'semanticversion', 'supportsautoscaling', 'zoneid', 'zonename', 'isoid', 'isoname', 'isostate', 'arch', 'mincpunumber', 'minmemory', 'supportsha', 'state', 'created', 'isourl'], tabs: [ { name: 'details', diff --git a/ui/src/config/section/infra/systemVms.js b/ui/src/config/section/infra/systemVms.js index 6e135ccdd36..4a5879b1762 100644 --- a/ui/src/config/section/infra/systemVms.js +++ b/ui/src/config/section/infra/systemVms.js @@ -26,7 +26,7 @@ export default { permission: ['listSystemVms'], searchFilters: ['name', 'zoneid', 'podid', 'hostid', 'systemvmtype', 'storageid', 'arch'], columns: ['name', 'state', 'agentstate', 'systemvmtype', 'publicip', 'privateip', 'linklocalip', 'version', 'hostname', 'arch', 'zonename'], - details: ['name', 'id', 'agentstate', 'systemvmtype', 'publicip', 'privateip', 'linklocalip', 'gateway', 'hostname', 'arch', 'version', 'zonename', 'created', 'activeviewersessions', 'isdynamicallyscalable', 'hostcontrolstate'], + details: ['name', 'id', 'agentstate', 'systemvmtype', 'publicip', 'privateip', 'linklocalip', 'gateway', 'hostname', 'arch', 'version', 'zonename', 'created', 'activeviewersessions', 'isdynamicallyscalable', 'hostcontrolstate', 'storageip'], resourceType: 'SystemVm', filters: () => { const filters = ['starting', 'running', 'stopping', 'stopped', 'destroyed', 'expunging', 'migrating', 'error', 'unknown', 'shutdown'] diff --git a/ui/src/config/section/network.js b/ui/src/config/section/network.js index 814eaf84f99..33b39d27172 100644 --- a/ui/src/config/section/network.js +++ b/ui/src/config/section/network.js @@ -356,7 +356,10 @@ export default { permission: ['listVnfAppliances'], resourceType: 'UserVm', params: () => { - return { details: 'servoff,tmpl,nics', isvnf: true } + return { + details: 'group,nics,secgrp,tmpl,servoff,diskoff,iso,volume,affgrp,backoff,vnfnics', + isvnf: true + } }, columns: () => { const fields = ['name', 'state', 'ipaddress'] diff --git a/ui/src/config/section/storage.js b/ui/src/config/section/storage.js index 41875ec4db5..75432314b03 100644 --- a/ui/src/config/section/storage.js +++ b/ui/src/config/section/storage.js @@ -92,7 +92,7 @@ export default { } ], searchFilters: () => { - var filters = ['name', 'zoneid', 'domainid', 'account', 'state', 'tags', 'serviceofferingid', 'diskofferingid', 'isencrypted'] + const filters = ['name', 'zoneid', 'domainid', 'account', 'state', 'tags', 'serviceofferingid', 'diskofferingid', 'isencrypted'] if (['Admin', 'DomainAdmin'].includes(store.getters.userInfo.roletype)) { filters.push('storageid') } @@ -311,7 +311,10 @@ export default { permission: ['listSnapshots'], resourceType: 'Snapshot', columns: () => { - var fields = ['name', 'state', 'volumename', 'intervaltype', 'physicalsize', 'created'] + const fields = ['name', 'state', 'volumename', 'intervaltype', 'physicalsize', 'created'] + if (store.getters.features.snapshotshowchainsize) { + fields.splice(fields.indexOf('created'), 0, 'chainsize', 'parentname') + } if (['Admin', 'DomainAdmin'].includes(store.getters.userInfo.roletype)) { fields.push('account') if (store.getters.listAllProjects) { @@ -324,7 +327,13 @@ export default { fields.push('zonename') return fields }, - details: ['name', 'id', 'volumename', 'volumetype', 'snapshottype', 'intervaltype', 'physicalsize', 'virtualsize', 'chainsize', 'account', 'domain', 'created'], + details: () => { + const fields = ['name', 'id', 'volumename', 'volumetype', 'snapshottype', 'intervaltype', 'physicalsize', 'virtualsize', 'account', 'domain', 'created'] + if (store.getters.features.snapshotshowchainsize) { + fields.splice(fields.indexOf('account'), 0, 'chainsize', 'parentname') + } + return fields + }, tabs: [ { name: 'details', @@ -346,7 +355,7 @@ export default { } ], searchFilters: () => { - var filters = ['name', 'domainid', 'account', 'tags', 'zoneid'] + const filters = ['name', 'domainid', 'account', 'tags', 'zoneid'] if (['Admin', 'DomainAdmin'].includes(store.getters.userInfo.roletype)) { filters.push('storageid') filters.push('imagestoreid') diff --git a/ui/src/config/section/user.js b/ui/src/config/section/user.js index a18994fd6ce..65c1a17f760 100644 --- a/ui/src/config/section/user.js +++ b/ui/src/config/section/user.js @@ -105,9 +105,10 @@ export default { message: 'message.enable.user', dataView: true, show: (record, store) => { - return ['Admin', 'DomainAdmin'].includes(store.userInfo.roletype) && !record.isdefault && - !(record.domain === 'ROOT' && record.account === 'admin' && record.accounttype === 1) && - ['disabled', 'locked'].includes(record.state) + if (!['disabled', 'locked'].includes(record.state) || record.isdefault || !['Admin', 'DomainAdmin'].includes(store.userInfo.roletype)) { + return false + } + return ![1, 4].includes(record.accounttype) || store.userInfo.roletype === 'Admin' } }, { @@ -117,9 +118,10 @@ export default { message: 'message.disable.user', dataView: true, show: (record, store) => { - return ['Admin', 'DomainAdmin'].includes(store.userInfo.roletype) && !record.isdefault && - !(record.domain === 'ROOT' && record.account === 'admin' && record.accounttype === 1) && - record.state === 'enabled' + if (record.state !== 'enabled' || record.isdefault || !['Admin', 'DomainAdmin'].includes(store.userInfo.roletype)) { + return false + } + return ![1, 4].includes(record.accounttype) || (store.userInfo.roletype === 'Admin' && record.id !== store.userInfo.id) } }, { @@ -131,9 +133,10 @@ export default { dataView: true, popup: true, show: (record, store) => { - return ['Admin', 'DomainAdmin'].includes(store.userInfo.roletype) && !record.isdefault && - !(record.domain === 'ROOT' && record.account === 'admin' && record.accounttype === 1) && - record.state === 'enabled' + if (record.state !== 'enabled' || record.isdefault || !['Admin', 'DomainAdmin'].includes(store.userInfo.roletype)) { + return false + } + return ![1, 4].includes(record.accounttype) || (store.userInfo.roletype === 'Admin' && record.id !== store.userInfo.id) } }, { diff --git a/ui/src/utils/acsrepo/index.js b/ui/src/utils/acsrepo/index.js new file mode 100644 index 00000000000..809bd7f1748 --- /dev/null +++ b/ui/src/utils/acsrepo/index.js @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +const BASE_KUBERNETES_ISO_URL = 'https://download.cloudstack.org/cks/' + +function getDefaultLatestKubernetesIsoParams (arch) { + return { + name: 'v1.33.1-calico-' + arch, + semanticversion: '1.33.1', + url: BASE_KUBERNETES_ISO_URL + 'setup-v1.33.1-calico-' + arch + '.iso', + arch: arch, + mincpunumber: 2, + minmemory: 2048 + } +} + +/** + * Returns the latest Kubernetes ISO info for the given architecture. + * Falls back to a hardcoded default if fetching fails. + * @param {string} arch + * @returns {Promise<{name: string, semanticversion: string, url: string, arch: string}>} + */ +export async function getLatestKubernetesIsoParams (arch) { + arch = arch || 'x86_64' + try { + const html = await fetch(BASE_KUBERNETES_ISO_URL, { cache: 'no-store' }).then(r => r.text()) + + const hrefs = [...html.matchAll(/href="([^"]+\.iso)"/gi)].map(m => m[1]) + + // Prefer files that explicitly include the arch (e.g. ...-x86_64.iso) + let isoHrefs = hrefs.filter(h => new RegExp(`${arch}\\.iso$`, 'i').test(h)) + + // Fallback: older files without arch suffix (e.g. setup-1.28.4.iso) + if (isoHrefs.length === 0) { + isoHrefs = hrefs.filter(h => /setup-\d+\.\d+\.\d+\.iso$/i.test(h)) + } + + const entries = isoHrefs.map(h => { + const m = h.match(/setup-(?:v)?(\d+\.\d+\.\d+)(?:-calico)?(?:-(x86_64|arm64))?/i) + return m + ? { + name: h.replace('.iso', ''), + semanticversion: m[1], + url: new URL(h, BASE_KUBERNETES_ISO_URL).toString(), + arch: m[2] || arch, + mincpunumber: 2, + minmemory: 2048 + } + : null + }).filter(Boolean) + + if (entries.length === 0) throw new Error('No matching ISOs found') + + entries.sort((a, b) => { + const pa = a.semanticversion.split('.').map(Number) + const pb = b.semanticversion.split('.').map(Number) + for (let i = 0; i < 3; i++) { + if ((pb[i] ?? 0) !== (pa[i] ?? 0)) return (pb[i] ?? 0) - (pa[i] ?? 0) + } + return 0 + }) + + return entries[0] + } catch { + return { ...getDefaultLatestKubernetesIsoParams(arch) } + } +} diff --git a/ui/src/utils/plugins.js b/ui/src/utils/plugins.js index a6e784aa8f7..b8856c7c9b0 100644 --- a/ui/src/utils/plugins.js +++ b/ui/src/utils/plugins.js @@ -224,18 +224,19 @@ export const notifierPlugin = { if (error.response.status) { msg = `${i18n.global.t('message.request.failed')} (${error.response.status})` } - if (error.message) { - desc = error.message - } - if (error.response.headers && 'x-description' in error.response.headers) { + if (error.response.headers?.['x-description']) { desc = error.response.headers['x-description'] - } - if (desc === '' && error.response.data) { + } else if (error.response.data) { const responseKey = _.findKey(error.response.data, 'errortext') if (responseKey) { desc = error.response.data[responseKey].errortext + } else if (typeof error.response.data === 'string') { + desc = error.response.data } } + if (!desc && error.message) { + desc = error.message + } } let countNotify = store.getters.countNotify countNotify++ @@ -549,6 +550,17 @@ export const dialogUtilPlugin = { onOk: () => callback(configRecord) }) } + + app.config.globalProperties.$notifyConfigurationValueChange = function (configRecord) { + if (!configRecord || configRecord.isdynamic || store.getters.userInfo?.roletype !== 'Admin') { + return + } + const server = configRecord.group === 'Usage Server' ? 'usage' : 'mgmt' + this.$notification.warning({ + message: this.$t('label.status'), + description: this.$t('message.restart.' + server + '.server') + }) + } } } diff --git a/ui/src/views/AutogenView.vue b/ui/src/views/AutogenView.vue index e0583cd97a4..f4aa842d8f2 100644 --- a/ui/src/views/AutogenView.vue +++ b/ui/src/views/AutogenView.vue @@ -17,7 +17,10 @@ @@ -83,6 +83,9 @@ export default { return 'light-row' } return 'dark-row' + }, + handleConfigRefresh (name, updatedRecord) { + this.$emit('refresh-config', name, updatedRecord) } } } diff --git a/ui/src/views/setting/ConfigurationTab.vue b/ui/src/views/setting/ConfigurationTab.vue index 75905cbd174..65b256c94c9 100644 --- a/ui/src/views/setting/ConfigurationTab.vue +++ b/ui/src/views/setting/ConfigurationTab.vue @@ -58,7 +58,8 @@ :count="count" :page="page" :pagesize="pagesize" - @change-page="changePage" /> + @change-page="changePage" + @refresh-config="handleConfigRefresh" /> + :config="config" + @refresh-config="handleConfigRefresh" /> @@ -322,6 +324,13 @@ export default { '#' + this.$route.path ) } + }, + handleConfigRefresh (name, updatedRecord) { + if (!name || !updatedRecord) return + const index = this.config.findIndex(item => item.name === name) + if (index !== -1) { + this.config.splice(index, 1, updatedRecord) + } } } } diff --git a/ui/src/views/setting/ConfigurationTable.vue b/ui/src/views/setting/ConfigurationTable.vue index da05b9342a0..7edc1b1aad6 100644 --- a/ui/src/views/setting/ConfigurationTable.vue +++ b/ui/src/views/setting/ConfigurationTable.vue @@ -32,7 +32,10 @@ {{record.displaytext }} {{ ' (' + record.name + ')' }}
    {{ record.description }} @@ -113,6 +116,9 @@ export default { return 'config-light-row' } return 'config-dark-row' + }, + handleConfigRefresh (name, updatedRecord) { + this.$emit('refresh-config', name, updatedRecord) } } } diff --git a/ui/src/views/setting/ConfigurationValue.vue b/ui/src/views/setting/ConfigurationValue.vue index 662e5ef142e..d9e36d9af53 100644 --- a/ui/src/views/setting/ConfigurationValue.vue +++ b/ui/src/views/setting/ConfigurationValue.vue @@ -187,7 +187,7 @@ @onClick="$resetConfigurationValueConfirm(configrecord, resetConfigurationValue)" v-if="editableValueKey === null" icon="reload-outlined" - :disabled="(!('resetConfiguration' in $store.getters.apis) || configDisabled || valueLoading)" /> + :disabled="(!('resetConfiguration' in $store.getters.apis) || configDisabled || valueLoading || configrecord.value === configrecord.defaultvalue)" />     @@ -273,6 +273,7 @@ export default { this.editableValueKey = null }, updateConfigurationValue (configrecord) { + let configRecordEntry = this.configrecord this.valueLoading = true this.editableValueKey = null var newValue = this.editableValue @@ -294,20 +295,13 @@ export default { params[this.scopeKey] = this.resource?.id } postAPI('updateConfiguration', params).then(json => { - this.editableValue = this.getEditableValue(json.updateconfigurationresponse.configuration) + configRecordEntry = json.updateconfigurationresponse.configuration + this.editableValue = this.getEditableValue(configRecordEntry) this.actualValue = this.editableValue this.$emit('change-config', { value: newValue }) this.$store.dispatch('RefreshFeatures') this.$messageConfigSuccess(`${this.$t('message.setting.updated')} ${configrecord.name}`, configrecord) - if (json.updateconfigurationresponse && - json.updateconfigurationresponse.configuration && - !json.updateconfigurationresponse.configuration.isdynamic && - ['Admin'].includes(this.$store.getters.userInfo.roletype)) { - this.$notification.warning({ - message: this.$t('label.status'), - description: this.$t('message.restart.mgmt.server') - }) - } + this.$notifyConfigurationValueChange(json?.updateconfigurationresponse?.configuration || null) }).catch(error => { this.editableValue = this.actualValue console.error(error) @@ -318,10 +312,11 @@ export default { }) }).finally(() => { this.valueLoading = false - this.$emit('refresh') + this.$emit('refresh', configrecord.name, configRecordEntry) }) }, resetConfigurationValue (configrecord) { + let configRecordEntry = this.configrecord this.valueLoading = true this.editableValueKey = null const params = { @@ -332,7 +327,8 @@ export default { params[this.scopeKey] = this.resource?.id } postAPI('resetConfiguration', params).then(json => { - this.editableValue = this.getEditableValue(json.resetconfigurationresponse.configuration) + configRecordEntry = json.resetconfigurationresponse.configuration + this.editableValue = this.getEditableValue(configRecordEntry) this.actualValue = this.editableValue var newValue = this.editableValue if (configrecord.type === 'Range') { @@ -341,15 +337,7 @@ export default { this.$emit('change-config', { value: newValue }) this.$store.dispatch('RefreshFeatures') this.$messageConfigSuccess(`${this.$t('label.setting')} ${configrecord.name} ${this.$t('label.reset.config.value')}`, configrecord) - if (json.resetconfigurationresponse && - json.resetconfigurationresponse.configuration && - !json.resetconfigurationresponse.configuration.isdynamic && - ['Admin'].includes(this.$store.getters.userInfo.roletype)) { - this.$notification.warning({ - message: this.$t('label.status'), - description: this.$t('message.restart.mgmt.server') - }) - } + this.$notifyConfigurationValueChange(json?.resetconfigurationresponse?.configuration || null) }).catch(error => { this.editableValue = this.actualValue console.error(error) @@ -360,7 +348,7 @@ export default { }) }).finally(() => { this.valueLoading = false - this.$emit('refresh') + this.$emit('refresh', configrecord.name, configRecordEntry) }) }, getEditableValue (configrecord) { diff --git a/ui/src/views/storage/CreateTemplate.vue b/ui/src/views/storage/CreateTemplate.vue index 81f0e3e782a..016af2662e6 100644 --- a/ui/src/views/storage/CreateTemplate.vue +++ b/ui/src/views/storage/CreateTemplate.vue @@ -73,42 +73,32 @@ - - - - - - {{ opt.path || opt.name || opt.description }} - - - + @change-option-value="handleDomainChange" /> - - - {{ acc.name }} - - + api="listAccounts" + :apiParams="accountsApiParams" + resourceType="account" + optionValueKey="name" + optionLabelKey="name" + defaultIcon="team-outlined" + allowClear="true" + :placeholder="apiParams.account.description" /> { - this.domains = json.listdomainsresponse.domain - }).finally(() => { - this.domainLoading = false - this.handleDomainChange(null) - }) - }, - async handleDomainChange (domain) { - this.domainid = domain + handleDomainChange (domainId) { + this.domainid = domainId this.form.account = null this.account = null - if ('listAccounts' in this.$store.getters.apis) { - await this.fetchAccounts() - } }, - fetchAccounts () { - return new Promise((resolve, reject) => { - getAPI('listAccounts', { - domainid: this.domainid - }).then(response => { - this.accounts = response?.listaccountsresponse?.account || [] - resolve(this.accounts) - }).catch(error => { - this.$notifyError(error) - }) - }) - }, - handleAccountChange (acc) { - if (acc) { - this.account = acc.name + handleAccountChange (accountName) { + if (accountName) { + this.account = accountName } else { - this.account = acc + this.account = null } }, handleSubmit (e) { diff --git a/ui/src/views/storage/UploadLocalVolume.vue b/ui/src/views/storage/UploadLocalVolume.vue index a6125fa514e..5e6a5b4fc27 100644 --- a/ui/src/views/storage/UploadLocalVolume.vue +++ b/ui/src/views/storage/UploadLocalVolume.vue @@ -57,43 +57,33 @@ - - - - - - {{ zone.name || zone.description }} - - - + api="listZones" + :apiParams="zonesApiParams" + resourceType="zone" + optionValueKey="id" + optionLabelKey="name" + defaultIcon="global-outlined" + selectFirstOption="true" + @change-option-value="handleZoneChange" /> - - - {{ offering.displaytext || offering.name }} - - + @change-option="onChangeDiskOffering" />