diff --git a/agent/pom.xml b/agent/pom.xml index 9caa6d992c8..76c37cd87be 100644 --- a/agent/pom.xml +++ b/agent/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT diff --git a/agent/src/main/java/com/cloud/agent/resource/consoleproxy/ConsoleProxyResource.java b/agent/src/main/java/com/cloud/agent/resource/consoleproxy/ConsoleProxyResource.java index ccd0d976e58..26f9d4b3d73 100644 --- a/agent/src/main/java/com/cloud/agent/resource/consoleproxy/ConsoleProxyResource.java +++ b/agent/src/main/java/com/cloud/agent/resource/consoleproxy/ConsoleProxyResource.java @@ -397,9 +397,8 @@ public class ConsoleProxyResource extends ServerResourceBase implements ServerRe } public String authenticateConsoleAccess(String host, String port, String vmId, String sid, String ticket, - Boolean isReauthentication, String sessionToken) { - - ConsoleAccessAuthenticationCommand cmd = new ConsoleAccessAuthenticationCommand(host, port, vmId, sid, ticket, sessionToken); + Boolean isReauthentication, String sessionToken, String clientAddress) { + ConsoleAccessAuthenticationCommand cmd = new ConsoleAccessAuthenticationCommand(host, port, vmId, sid, ticket, sessionToken, clientAddress); cmd.setReauthenticating(isReauthentication); ConsoleProxyAuthenticationResult result = new ConsoleProxyAuthenticationResult(); diff --git a/api/pom.xml b/api/pom.xml index 32897725e0c..ec68e24c7e5 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT diff --git a/api/src/main/java/com/cloud/agent/api/to/PortForwardingRuleTO.java b/api/src/main/java/com/cloud/agent/api/to/PortForwardingRuleTO.java index 76d6d952814..d43625c09a9 100644 --- a/api/src/main/java/com/cloud/agent/api/to/PortForwardingRuleTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/PortForwardingRuleTO.java @@ -19,6 +19,9 @@ package com.cloud.agent.api.to; import com.cloud.network.rules.FirewallRule; import com.cloud.network.rules.PortForwardingRule; import com.cloud.utils.net.NetUtils; +import org.apache.commons.lang3.StringUtils; + +import java.util.List; /** * PortForwardingRuleTO specifies one port forwarding rule. @@ -29,6 +32,8 @@ public class PortForwardingRuleTO extends FirewallRuleTO { String dstIp; int[] dstPortRange; + List sourceCidrList; + protected PortForwardingRuleTO() { super(); } @@ -37,6 +42,7 @@ public class PortForwardingRuleTO extends FirewallRuleTO { super(rule, srcVlanTag, srcIp); this.dstIp = rule.getDestinationIpAddress().addr(); this.dstPortRange = new int[] {rule.getDestinationPortStart(), rule.getDestinationPortEnd()}; + this.sourceCidrList = rule.getSourceCidrList(); } public PortForwardingRuleTO(long id, String srcIp, int srcPortStart, int srcPortEnd, String dstIp, int dstPortStart, int dstPortEnd, String protocol, @@ -58,4 +64,11 @@ public class PortForwardingRuleTO extends FirewallRuleTO { return NetUtils.portRangeToString(dstPortRange); } + public String getSourceCidrListAsString() { + if (sourceCidrList != null) { + return StringUtils.join(sourceCidrList, ","); + } + return null; + } + } diff --git a/api/src/main/java/com/cloud/bgp/BGPService.java b/api/src/main/java/com/cloud/bgp/BGPService.java index 935237092dd..61d149f2847 100644 --- a/api/src/main/java/com/cloud/bgp/BGPService.java +++ b/api/src/main/java/com/cloud/bgp/BGPService.java @@ -21,6 +21,7 @@ import com.cloud.network.Network; import com.cloud.network.vpc.Vpc; import com.cloud.utils.Pair; import org.apache.cloudstack.api.command.user.bgp.ListASNumbersCmd; +import org.apache.cloudstack.network.BgpPeer; import java.util.List; @@ -36,4 +37,8 @@ public interface BGPService { boolean applyBgpPeers(Network network, boolean continueOnError) throws ResourceUnavailableException; boolean applyBgpPeers(Vpc vpc, boolean continueOnError) throws ResourceUnavailableException; + + List getBgpPeersForNetwork(Network network); + + List getBgpPeersForVpc(Vpc vpc); } diff --git a/api/src/main/java/com/cloud/event/EventTypes.java b/api/src/main/java/com/cloud/event/EventTypes.java index 5e5309965c1..81ed185dae5 100644 --- a/api/src/main/java/com/cloud/event/EventTypes.java +++ b/api/src/main/java/com/cloud/event/EventTypes.java @@ -292,6 +292,7 @@ public class EventTypes { //register for user API and secret keys public static final String EVENT_REGISTER_FOR_SECRET_API_KEY = "REGISTER.USER.KEY"; + public static final String API_KEY_ACCESS_UPDATE = "API.KEY.ACCESS.UPDATE"; // Template Events public static final String EVENT_TEMPLATE_CREATE = "TEMPLATE.CREATE"; diff --git a/api/src/main/java/com/cloud/hypervisor/Hypervisor.java b/api/src/main/java/com/cloud/hypervisor/Hypervisor.java index e1108b34a83..27ffef1c370 100644 --- a/api/src/main/java/com/cloud/hypervisor/Hypervisor.java +++ b/api/src/main/java/com/cloud/hypervisor/Hypervisor.java @@ -20,37 +20,57 @@ import com.cloud.storage.Storage.ImageFormat; import org.apache.commons.lang3.StringUtils; import java.util.LinkedHashMap; +import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.Set; +import java.util.EnumSet; +import java.util.stream.Collectors; + +import static com.cloud.hypervisor.Hypervisor.HypervisorType.Functionality.DirectDownloadTemplate; +import static com.cloud.hypervisor.Hypervisor.HypervisorType.Functionality.RootDiskSizeOverride; +import static com.cloud.hypervisor.Hypervisor.HypervisorType.Functionality.VmStorageMigration; public class Hypervisor { public static class HypervisorType { + public enum Functionality { + DirectDownloadTemplate, + RootDiskSizeOverride, + VmStorageMigration + } + private static final Map hypervisorTypeMap = new LinkedHashMap<>(); public static final HypervisorType None = new HypervisorType("None"); //for storage hosts - public static final HypervisorType XenServer = new HypervisorType("XenServer", ImageFormat.VHD); - public static final HypervisorType KVM = new HypervisorType("KVM", ImageFormat.QCOW2); - public static final HypervisorType VMware = new HypervisorType("VMware", ImageFormat.OVA); + public static final HypervisorType XenServer = new HypervisorType("XenServer", ImageFormat.VHD, EnumSet.of(RootDiskSizeOverride, VmStorageMigration)); + public static final HypervisorType KVM = new HypervisorType("KVM", ImageFormat.QCOW2, EnumSet.of(DirectDownloadTemplate, RootDiskSizeOverride, VmStorageMigration)); + public static final HypervisorType VMware = new HypervisorType("VMware", ImageFormat.OVA, EnumSet.of(RootDiskSizeOverride, VmStorageMigration)); public static final HypervisorType Hyperv = new HypervisorType("Hyperv"); public static final HypervisorType VirtualBox = new HypervisorType("VirtualBox"); public static final HypervisorType Parralels = new HypervisorType("Parralels"); public static final HypervisorType BareMetal = new HypervisorType("BareMetal"); - public static final HypervisorType Simulator = new HypervisorType("Simulator"); + public static final HypervisorType Simulator = new HypervisorType("Simulator", null, EnumSet.of(RootDiskSizeOverride, VmStorageMigration)); public static final HypervisorType Ovm = new HypervisorType("Ovm", ImageFormat.RAW); public static final HypervisorType Ovm3 = new HypervisorType("Ovm3", ImageFormat.RAW); public static final HypervisorType LXC = new HypervisorType("LXC"); - public static final HypervisorType Custom = new HypervisorType("Custom"); + public static final HypervisorType Custom = new HypervisorType("Custom", null, EnumSet.of(RootDiskSizeOverride)); public static final HypervisorType Any = new HypervisorType("Any"); /*If you don't care about the hypervisor type*/ private final String name; private final ImageFormat imageFormat; + private final Set supportedFunctionalities; public HypervisorType(String name) { - this(name, null); + this(name, null, EnumSet.noneOf(Functionality.class)); } public HypervisorType(String name, ImageFormat imageFormat) { + this(name, imageFormat, EnumSet.noneOf(Functionality.class)); + } + + public HypervisorType(String name, ImageFormat imageFormat, Set supportedFunctionalities) { this.name = name; this.imageFormat = imageFormat; + this.supportedFunctionalities = supportedFunctionalities; if (name.equals("Parralels")){ // typo in the original code hypervisorTypeMap.put("parallels", this); } else { @@ -81,6 +101,12 @@ public class Hypervisor { return hypervisorType; } + public static List getListOfHypervisorsSupportingFunctionality(Functionality functionality) { + return hypervisorTypeMap.values().stream() + .filter(hypervisor -> hypervisor.supportedFunctionalities.contains(functionality)) + .collect(Collectors.toList()); + } + /** * Returns the display name of a hypervisor type in case the custom hypervisor is used, * using the 'hypervisor.custom.display.name' setting. Otherwise, returns hypervisor name @@ -102,6 +128,15 @@ public class Hypervisor { return name; } + /** + * Make this method to be part of the properties of the hypervisor type itself. + * + * @return true if the hypervisor plugin support the specified functionality + */ + public boolean isFunctionalitySupported(Functionality functionality) { + return supportedFunctionalities.contains(functionality); + } + @Override public int hashCode() { return Objects.hash(name); diff --git a/api/src/main/java/com/cloud/network/rules/RulesService.java b/api/src/main/java/com/cloud/network/rules/RulesService.java index 0b4afeef945..547d4ab51e0 100644 --- a/api/src/main/java/com/cloud/network/rules/RulesService.java +++ b/api/src/main/java/com/cloud/network/rules/RulesService.java @@ -26,6 +26,7 @@ import com.cloud.exception.ResourceUnavailableException; import com.cloud.user.Account; import com.cloud.utils.Pair; import com.cloud.utils.net.Ip; +import org.apache.cloudstack.api.command.user.firewall.UpdatePortForwardingRuleCmd; public interface RulesService { Pair, Integer> searchStaticNatRules(Long ipId, Long id, Long vmId, Long start, Long size, String accountName, Long domainId, @@ -81,6 +82,8 @@ public interface RulesService { boolean disableStaticNat(long ipId) throws ResourceUnavailableException, NetworkRuleConflictException, InsufficientAddressCapacityException; - PortForwardingRule updatePortForwardingRule(long id, Integer privatePort, Integer privateEndPort, Long virtualMachineId, Ip vmGuestIp, String customId, Boolean forDisplay); + PortForwardingRule updatePortForwardingRule(UpdatePortForwardingRuleCmd cmd); + + void validatePortForwardingSourceCidrList(List sourceCidrList); } diff --git a/api/src/main/java/com/cloud/server/ManagementServerHostStats.java b/api/src/main/java/com/cloud/server/ManagementServerHostStats.java index 1f201d7689f..1eea7addba3 100644 --- a/api/src/main/java/com/cloud/server/ManagementServerHostStats.java +++ b/api/src/main/java/com/cloud/server/ManagementServerHostStats.java @@ -32,6 +32,8 @@ public interface ManagementServerHostStats { String getManagementServerHostUuid(); + long getManagementServerRunId(); + long getSessions(); double getCpuUtilization(); diff --git a/api/src/main/java/com/cloud/user/Account.java b/api/src/main/java/com/cloud/user/Account.java index bb9838f137a..6be4d0a48f6 100644 --- a/api/src/main/java/com/cloud/user/Account.java +++ b/api/src/main/java/com/cloud/user/Account.java @@ -93,4 +93,8 @@ public interface Account extends ControlledEntity, InternalIdentity, Identity { boolean isDefault(); + public void setApiKeyAccess(Boolean apiKeyAccess); + + public Boolean getApiKeyAccess(); + } diff --git a/api/src/main/java/com/cloud/user/AccountService.java b/api/src/main/java/com/cloud/user/AccountService.java index 60db7abb734..e2c3bed0c29 100644 --- a/api/src/main/java/com/cloud/user/AccountService.java +++ b/api/src/main/java/com/cloud/user/AccountService.java @@ -19,6 +19,7 @@ package com.cloud.user; import java.util.List; import java.util.Map; +import com.cloud.utils.Pair; import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.acl.SecurityChecker.AccessType; @@ -127,9 +128,9 @@ public interface AccountService { */ UserAccount getUserAccountById(Long userId); - public Map getKeys(GetUserKeysCmd cmd); + public Pair> getKeys(GetUserKeysCmd cmd); - public Map getKeys(Long userId); + public Pair> getKeys(Long userId); /** * Lists user two-factor authentication provider plugins diff --git a/api/src/main/java/com/cloud/user/User.java b/api/src/main/java/com/cloud/user/User.java index 422e264f10b..041b39ad272 100644 --- a/api/src/main/java/com/cloud/user/User.java +++ b/api/src/main/java/com/cloud/user/User.java @@ -94,4 +94,9 @@ public interface User extends OwnedBy, InternalIdentity { public boolean isUser2faEnabled(); public String getKeyFor2fa(); + + public void setApiKeyAccess(Boolean apiKeyAccess); + + public Boolean getApiKeyAccess(); + } diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index bb16b0ff90d..bf8b79b29d0 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -35,6 +35,7 @@ public class ApiConstants { public static final String ALLOW_USER_FORCE_STOP_VM = "allowuserforcestopvm"; public static final String ANNOTATION = "annotation"; public static final String API_KEY = "apikey"; + public static final String API_KEY_ACCESS = "apikeyaccess"; public static final String ARCHIVED = "archived"; public static final String ARCH = "arch"; public static final String AS_NUMBER = "asnumber"; @@ -243,6 +244,7 @@ public class ApiConstants { public static final String ICMP_TYPE = "icmptype"; public static final String ID = "id"; public static final String IDS = "ids"; + public static final String IMPORT_INSTANCE_HOST_ID = "importinstancehostid"; public static final String INDEX = "index"; public static final String INSTANCES_DISKS_STATS_RETENTION_ENABLED = "instancesdisksstatsretentionenabled"; public static final String INSTANCES_DISKS_STATS_RETENTION_TIME = "instancesdisksstatsretentiontime"; @@ -381,6 +383,14 @@ public class ApiConstants { public static final String PATH = "path"; public static final String PAYLOAD = "payload"; public static final String PAYLOAD_URL = "payloadurl"; + public static final String PEERS = "peers"; + public static final String PEER_ID = "peerid"; + public static final String PEER_NAME = "peername"; + public static final String PEER_MSID = "peermsid"; + public static final String PEER_RUNID = "peerrunid"; + public static final String PEER_SERVICE_IP = "peerserviceip"; + public static final String PEER_SERVICE_PORT = "peerserviceport"; + public static final String PEER_STATE = "peerstate"; public static final String POD_ID = "podid"; public static final String POD_NAME = "podname"; public static final String POD_IDS = "podids"; @@ -455,6 +465,7 @@ public class ApiConstants { public static final String SNAPSHOT_POLICY_ID = "snapshotpolicyid"; public static final String SNAPSHOT_TYPE = "snapshottype"; public static final String SNAPSHOT_QUIESCEVM = "quiescevm"; + public static final String SOURCE_CIDR_LIST = "sourcecidrlist"; public static final String SOURCE_ZONE_ID = "sourcezoneid"; public static final String SSL_VERIFICATION = "sslverification"; public static final String START_ASN = "startasn"; @@ -986,6 +997,7 @@ public class ApiConstants { public static final String ACL_NAME = "aclname"; public static final String NUMBER = "number"; public static final String IS_DYNAMICALLY_SCALABLE = "isdynamicallyscalable"; + public static final String ROUTED_MODE_ENABLED = "routedmodeenabled"; public static final String ROUTING = "isrouting"; public static final String ROUTING_MODE = "routingmode"; public static final String MAX_CONNECTIONS = "maxconnections"; @@ -1238,4 +1250,30 @@ public class ApiConstants { public enum DomainDetails { all, resource, min; } + + public enum ApiKeyAccess { + DISABLED(false), + ENABLED(true), + INHERIT(null); + + Boolean apiKeyAccess; + + ApiKeyAccess(Boolean keyAccess) { + apiKeyAccess = keyAccess; + } + + public Boolean toBoolean() { + return apiKeyAccess; + } + + public static ApiKeyAccess fromBoolean(Boolean value) { + if (value == null) { + return INHERIT; + } else if (value) { + return ENABLED; + } else { + return DISABLED; + } + } + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/account/UpdateAccountCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/account/UpdateAccountCmd.java index 91cbb90e4da..3347a0d09f3 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/account/UpdateAccountCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/account/UpdateAccountCmd.java @@ -21,7 +21,9 @@ import java.util.Map; import javax.inject.Inject; +import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.command.user.UserCmd; import org.apache.cloudstack.api.response.RoleResponse; import org.apache.cloudstack.acl.SecurityChecker.AccessType; @@ -40,8 +42,8 @@ import org.apache.cloudstack.region.RegionService; import com.cloud.user.Account; @APICommand(name = "updateAccount", description = "Updates account information for the authenticated user", responseObject = AccountResponse.class, entityType = {Account.class}, - requestHasSensitiveInfo = false, responseHasSensitiveInfo = true) -public class UpdateAccountCmd extends BaseCmd { + responseView = ResponseView.Restricted, requestHasSensitiveInfo = false, responseHasSensitiveInfo = true) +public class UpdateAccountCmd extends BaseCmd implements UserCmd { ///////////////////////////////////////////////////// //////////////// API parameters ///////////////////// @@ -70,6 +72,9 @@ public class UpdateAccountCmd extends BaseCmd { @Parameter(name = ApiConstants.ACCOUNT_DETAILS, type = CommandType.MAP, description = "Details for the account used to store specific parameters") private Map details; + @Parameter(name = ApiConstants.API_KEY_ACCESS, type = CommandType.STRING, description = "Determines if Api key access for this user is enabled, disabled or inherits the value from its parent, the domain level setting api.key.access", since = "4.20.1.0", authorized = {RoleType.Admin}) + private String apiKeyAccess; + @Inject RegionService _regionService; @@ -109,6 +114,10 @@ public class UpdateAccountCmd extends BaseCmd { return params; } + public String getApiKeyAccess() { + return apiKeyAccess; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @@ -131,7 +140,7 @@ public class UpdateAccountCmd extends BaseCmd { public void execute() { Account result = _regionService.updateAccount(this); if (result != null){ - AccountResponse response = _responseGenerator.createAccountResponse(ResponseView.Full, result); + AccountResponse response = _responseGenerator.createAccountResponse(getResponseView(), result); response.setResponseName(getCommandName()); setResponseObject(response); } else { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/management/ListMgmtsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/management/ListMgmtsCmd.java index a68ed62857a..7b7eae1d0e9 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/management/ListMgmtsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/management/ListMgmtsCmd.java @@ -23,6 +23,7 @@ import org.apache.cloudstack.api.BaseListCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.ManagementServerResponse; +import org.apache.commons.lang3.BooleanUtils; @APICommand(name = "listManagementServers", description = "Lists management servers.", responseObject = ManagementServerResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) @@ -39,6 +40,11 @@ public class ListMgmtsCmd extends BaseListCmd { @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, description = "the name of the management server") private String hostName; + @Parameter(name = ApiConstants.PEERS, type = CommandType.BOOLEAN, + description = "Whether to return the management server peers or not. By default, the management server peers will not be returned.", + since = "4.20.1.0") + private Boolean peers; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -51,6 +57,10 @@ public class ListMgmtsCmd extends BaseListCmd { return hostName; } + public Boolean getPeers() { + return BooleanUtils.toBooleanDefaultIfNull(peers, false); + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/user/GetUserKeysCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/user/GetUserKeysCmd.java index 3a3414d95d8..cdd239f72b5 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/user/GetUserKeysCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/user/GetUserKeysCmd.java @@ -20,6 +20,7 @@ package org.apache.cloudstack.api.command.admin.user; import com.cloud.user.Account; import com.cloud.user.User; +import com.cloud.utils.Pair; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; @@ -54,11 +55,13 @@ public class GetUserKeysCmd extends BaseCmd{ else return Account.ACCOUNT_ID_SYSTEM; } public void execute(){ - Map keys = _accountService.getKeys(this); + Pair> keys = _accountService.getKeys(this); + RegisterResponse response = new RegisterResponse(); if(keys != null){ - response.setApiKey(keys.get("apikey")); - response.setSecretKey(keys.get("secretkey")); + response.setApiKeyAccess(keys.first()); + response.setApiKey(keys.second().get("apikey")); + response.setSecretKey(keys.second().get("secretkey")); } response.setObjectName("userkeys"); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/user/ListUsersCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/user/ListUsersCmd.java index ef9e3fa2240..27a78c738c9 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/user/ListUsersCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/user/ListUsersCmd.java @@ -19,20 +19,23 @@ package org.apache.cloudstack.api.command.admin.user; import com.cloud.server.ResourceIcon; import com.cloud.server.ResourceTag; import com.cloud.user.Account; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.command.user.UserCmd; import org.apache.cloudstack.api.response.ResourceIconResponse; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseListAccountResourcesCmd; import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject.ResponseView; import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.UserResponse; import java.util.List; @APICommand(name = "listUsers", description = "Lists user accounts", responseObject = UserResponse.class, - requestHasSensitiveInfo = false, responseHasSensitiveInfo = true) -public class ListUsersCmd extends BaseListAccountResourcesCmd { + responseView = ResponseView.Restricted, requestHasSensitiveInfo = false, responseHasSensitiveInfo = true) +public class ListUsersCmd extends BaseListAccountResourcesCmd implements UserCmd { ///////////////////////////////////////////////////// @@ -53,6 +56,9 @@ public class ListUsersCmd extends BaseListAccountResourcesCmd { @Parameter(name = ApiConstants.USERNAME, type = CommandType.STRING, description = "List user by the username") private String username; + @Parameter(name = ApiConstants.API_KEY_ACCESS, type = CommandType.STRING, description = "List users by the Api key access value", since = "4.20.1.0", authorized = {RoleType.Admin}) + private String apiKeyAccess; + @Parameter(name = ApiConstants.SHOW_RESOURCE_ICON, type = CommandType.BOOLEAN, description = "flag to display the resource icon for users") private Boolean showIcon; @@ -77,6 +83,10 @@ public class ListUsersCmd extends BaseListAccountResourcesCmd { return username; } + public String getApiKeyAccess() { + return apiKeyAccess; + } + public Boolean getShowIcon() { return showIcon != null ? showIcon : false; } @@ -87,7 +97,7 @@ public class ListUsersCmd extends BaseListAccountResourcesCmd { @Override public void execute() { - ListResponse response = _queryService.searchForUsers(this); + ListResponse response = _queryService.searchForUsers(getResponseView(), this); response.setResponseName(getCommandName()); this.setResponseObject(response); if (response != null && response.getCount() > 0 && getShowIcon()) { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/user/UpdateUserCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/user/UpdateUserCmd.java index c9e1e934152..3d7f51ae220 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/user/UpdateUserCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/user/UpdateUserCmd.java @@ -18,6 +18,7 @@ package org.apache.cloudstack.api.command.admin.user; import javax.inject.Inject; +import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; @@ -69,6 +70,9 @@ public class UpdateUserCmd extends BaseCmd { @Parameter(name = ApiConstants.USER_SECRET_KEY, type = CommandType.STRING, description = "The secret key for the user. Must be specified with userApiKey") private String secretKey; + @Parameter(name = ApiConstants.API_KEY_ACCESS, type = CommandType.STRING, description = "Determines if Api key access for this user is enabled, disabled or inherits the value from its parent, the owning account", since = "4.20.1.0", authorized = {RoleType.Admin}) + private String apiKeyAccess; + @Parameter(name = ApiConstants.TIMEZONE, type = CommandType.STRING, description = "Specifies a timezone for this command. For more information on the timezone parameter, see Time Zone Format.") @@ -120,6 +124,10 @@ public class UpdateUserCmd extends BaseCmd { return secretKey; } + public String getApiKeyAccess() { + return apiKeyAccess; + } + public String getTimezone() { return timezone; } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java index 6f148ff0ee4..db43b53ab9a 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java @@ -144,15 +144,19 @@ public class ImportVmCmd extends ImportUnmanagedInstanceCmd { private String clusterName; @Parameter(name = ApiConstants.CONVERT_INSTANCE_HOST_ID, type = CommandType.UUID, entityType = HostResponse.class, - description = "(only for importing VMs from VMware to KVM) optional - the host to perform the virt-v2v migration from VMware to KVM.") + description = "(only for importing VMs from VMware to KVM) optional - the host to perform the virt-v2v conversion from VMware to KVM.") private Long convertInstanceHostId; + @Parameter(name = ApiConstants.IMPORT_INSTANCE_HOST_ID, type = CommandType.UUID, entityType = HostResponse.class, since = "4.19.2", + description = "(only for importing VMs from VMware to KVM) optional - the host to import the converted instance from VMware to KVM.") + private Long importInstanceHostId; + @Parameter(name = ApiConstants.CONVERT_INSTANCE_STORAGE_POOL_ID, type = CommandType.UUID, entityType = StoragePoolResponse.class, description = "(only for importing VMs from VMware to KVM) optional - the temporary storage pool to perform the virt-v2v migration from VMware to KVM.") private Long convertStoragePoolId; @Parameter(name = ApiConstants.FORCE_MS_TO_IMPORT_VM_FILES, type = CommandType.BOOLEAN, - description = "(only for importing VMs from VMware to KVM) optional - if true, forces MS to import VM file(s) to temporary storage, else uses KVM Host if ovftool is available, falls back to MS if not.") + description = "(only for importing VMs from VMware to KVM) optional - if true, forces MS to export OVF from VMware to temporary storage, else uses KVM Host if ovftool is available, falls back to MS if not.") private Boolean forceMsToImportVmFiles; ///////////////////////////////////////////////////// @@ -199,6 +203,10 @@ public class ImportVmCmd extends ImportUnmanagedInstanceCmd { return convertInstanceHostId; } + public Long getImportInstanceHostId() { + return importInstanceHostId; + } + public Long getConvertStoragePoolId() { return convertStoragePoolId; } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/account/ListAccountsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/account/ListAccountsCmd.java index 0a962b19e4f..9157188fdee 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/account/ListAccountsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/account/ListAccountsCmd.java @@ -20,6 +20,7 @@ import java.util.ArrayList; import java.util.EnumSet; import java.util.List; +import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; @@ -70,6 +71,9 @@ public class ListAccountsCmd extends BaseListDomainResourcesCmd implements UserC description = "comma separated list of account details requested, value can be a list of [ all, resource, min]") private List viewDetails; + @Parameter(name = ApiConstants.API_KEY_ACCESS, type = CommandType.STRING, description = "List accounts by the Api key access value", since = "4.20.1.0", authorized = {RoleType.Admin}) + private String apiKeyAccess; + @Parameter(name = ApiConstants.SHOW_RESOURCE_ICON, type = CommandType.BOOLEAN, description = "flag to display the resource icon for accounts") private Boolean showIcon; @@ -120,6 +124,10 @@ public class ListAccountsCmd extends BaseListDomainResourcesCmd implements UserC return dv; } + public String getApiKeyAccess() { + return apiKeyAccess; + } + public boolean getShowIcon() { return showIcon != null ? showIcon : false; } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/firewall/CreatePortForwardingRuleCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/firewall/CreatePortForwardingRuleCmd.java index 3545b3d21fb..9d1fe176019 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/firewall/CreatePortForwardingRuleCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/firewall/CreatePortForwardingRuleCmd.java @@ -105,8 +105,12 @@ public class CreatePortForwardingRuleCmd extends BaseAsyncCreateCmd implements P description = "the ID of the virtual machine for the port forwarding rule") private Long virtualMachineId; - @Parameter(name = ApiConstants.CIDR_LIST, type = CommandType.LIST, collectionType = CommandType.STRING, description = "the cidr list to forward traffic from. Multiple entries must be separated by a single comma character (,). This parameter is deprecated. Do not use.") - private List cidrlist; + @Parameter(name = ApiConstants.CIDR_LIST, + type = CommandType.LIST, + collectionType = CommandType.STRING, + description = " the source CIDR list to allow traffic from; all other CIDRs will be blocked. " + + "Multiple entries must be separated by a single comma character (,). This param will be used only for VPC tiers. By default, all CIDRs are allowed.") + private List sourceCidrList; @Parameter(name = ApiConstants.OPEN_FIREWALL, type = CommandType.BOOLEAN, description = "if true, firewall rule for source/end public port is automatically created; " + "if false - firewall rule has to be created explicitly. If not specified 1) defaulted to false when PF" @@ -155,11 +159,7 @@ public class CreatePortForwardingRuleCmd extends BaseAsyncCreateCmd implements P @Override public List getSourceCidrList() { - if (cidrlist != null) { - throw new InvalidParameterValueException("Parameter cidrList is deprecated; if you need to open firewall " - + "rule for the specific cidr, please refer to createFirewallRule command"); - } - return null; + return sourceCidrList; } public Boolean getOpenFirewall() { @@ -332,12 +332,6 @@ public class CreatePortForwardingRuleCmd extends BaseAsyncCreateCmd implements P @Override public void create() { - // cidr list parameter is deprecated - if (cidrlist != null) { - throw new InvalidParameterValueException( - "Parameter cidrList is deprecated; if you need to open firewall rule for the specific cidr, please refer to createFirewallRule command"); - } - Ip privateIp = getVmSecondaryIp(); if (privateIp != null) { if (!NetUtils.isValidIp4(privateIp.toString())) { @@ -345,6 +339,8 @@ public class CreatePortForwardingRuleCmd extends BaseAsyncCreateCmd implements P } } + _rulesService.validatePortForwardingSourceCidrList(sourceCidrList); + try { PortForwardingRule result = _rulesService.createPortForwardingRule(this, virtualMachineId, privateIp, getOpenFirewall(), isDisplay()); setEntityId(result.getId()); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/firewall/UpdatePortForwardingRuleCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/firewall/UpdatePortForwardingRuleCmd.java index 3fb66bd861f..ee4bd18ad40 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/firewall/UpdatePortForwardingRuleCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/firewall/UpdatePortForwardingRuleCmd.java @@ -32,6 +32,8 @@ import com.cloud.network.rules.PortForwardingRule; import com.cloud.user.Account; import com.cloud.utils.net.Ip; +import java.util.List; + @APICommand(name = "updatePortForwardingRule", responseObject = FirewallRuleResponse.class, description = "Updates a port forwarding rule. Only the private port and the virtual machine can be updated.", entityType = {PortForwardingRule.class}, @@ -63,6 +65,13 @@ public class UpdatePortForwardingRuleCmd extends BaseAsyncCustomIdCmd { @Parameter(name = ApiConstants.FOR_DISPLAY, type = CommandType.BOOLEAN, description = "an optional field, whether to the display the rule to the end user or not", since = "4.4", authorized = {RoleType.Admin}) private Boolean display; + @Parameter(name = ApiConstants.CIDR_LIST, + type = CommandType.LIST, + collectionType = CommandType.STRING, + description = " the source CIDR list to allow traffic from; all other CIDRs will be blocked. " + + "Multiple entries must be separated by a single comma character (,). This param will be used only for VPC tiers. By default, all CIDRs are allowed.") + private List sourceCidrList; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -94,6 +103,10 @@ public class UpdatePortForwardingRuleCmd extends BaseAsyncCustomIdCmd { return display; } + public List getSourceCidrList() { + return sourceCidrList; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @@ -130,7 +143,7 @@ public class UpdatePortForwardingRuleCmd extends BaseAsyncCustomIdCmd { @Override public void execute() { - PortForwardingRule rule = _rulesService.updatePortForwardingRule(getId(), getPrivatePort(), getPrivateEndPort(), getVirtualMachineId(), getVmGuestIp(), getCustomId(), getDisplay()); + PortForwardingRule rule = _rulesService.updatePortForwardingRule(this); FirewallRuleResponse fwResponse = new FirewallRuleResponse(); if (rule != null) { fwResponse = _responseGenerator.createPortForwardingRuleResponse(rule); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/loadbalancer/CreateLoadBalancerRuleCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/loadbalancer/CreateLoadBalancerRuleCmd.java index f86d1ae85da..34798c4efe1 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/loadbalancer/CreateLoadBalancerRuleCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/loadbalancer/CreateLoadBalancerRuleCmd.java @@ -104,7 +104,7 @@ public class CreateLoadBalancerRuleCmd extends BaseAsyncCreateCmd /*implements L @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, entityType = DomainResponse.class, description = "the domain ID associated with the load balancer") private Long domainId; - @Parameter(name = ApiConstants.CIDR_LIST, type = CommandType.LIST, collectionType = CommandType.STRING, since = "4.18.0.0", description = "the CIDR list to allow traffic, " + @Parameter(name = ApiConstants.CIDR_LIST, type = CommandType.LIST, collectionType = CommandType.STRING, since = "4.18.0.0", description = "the source CIDR list to allow traffic from; " + "all other CIDRs will be blocked. Multiple entries must be separated by a single comma character (,). By default, all CIDRs are allowed.") private List cidrlist; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java index a7826dedcd0..1f968b869b9 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java @@ -362,7 +362,9 @@ public class RegisterTemplateCmd extends BaseCmd implements UserCmd { "Parameter zoneids cannot combine all zones (-1) option with other zones"); String customHypervisor = HypervisorGuru.HypervisorCustomDisplayName.value(); - if (isDirectDownload() && !(getHypervisor().equalsIgnoreCase(Hypervisor.HypervisorType.KVM.toString()) + if (isDirectDownload() && + !(Hypervisor.HypervisorType.getType(getHypervisor()) + .isFunctionalitySupported(Hypervisor.HypervisorType.Functionality.DirectDownloadTemplate) || getHypervisor().equalsIgnoreCase(customHypervisor))) { throw new ServerApiException(ApiErrorCode.PARAM_ERROR, String.format("Parameter directdownload " + "is only allowed for KVM or %s templates", customHypervisor)); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/ResizeVolumeCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/ResizeVolumeCmd.java index 65a3d6a7063..eb89115cf46 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/ResizeVolumeCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/ResizeVolumeCmd.java @@ -73,6 +73,10 @@ public class ResizeVolumeCmd extends BaseAsyncCmd implements UserCmd { description = "new disk offering id") private Long newDiskOfferingId; + @Parameter(name = ApiConstants.AUTO_MIGRATE, type = CommandType.BOOLEAN, required = false, + description = "Flag to allow automatic migration of the volume to another suitable storage pool that accommodates the new size", since = "4.20.1") + private Boolean autoMigrate; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -129,6 +133,10 @@ public class ResizeVolumeCmd extends BaseAsyncCmd implements UserCmd { return newDiskOfferingId; } + public boolean getAutoMigrate() { + return autoMigrate == null ? false : autoMigrate; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/response/AccountResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/AccountResponse.java index 7a84e85a4a6..6fc098295f6 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/AccountResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/AccountResponse.java @@ -271,6 +271,10 @@ public class AccountResponse extends BaseResponse implements ResourceLimitAndCou @Param(description = "The tagged resource limit and count for the account", since = "4.20.0") List taggedResources; + @SerializedName(ApiConstants.API_KEY_ACCESS) + @Param(description = "whether api key access is Enabled, Disabled or set to Inherit (it inherits the value from the parent)", since = "4.20.1.0") + ApiConstants.ApiKeyAccess apiKeyAccess; + @Override public String getObjectId() { return id; @@ -554,4 +558,8 @@ public class AccountResponse extends BaseResponse implements ResourceLimitAndCou public void setTaggedResourceLimitsAndCounts(List taggedResourceLimitsAndCounts) { this.taggedResources = taggedResourceLimitsAndCounts; } + + public void setApiKeyAccess(Boolean apiKeyAccess) { + this.apiKeyAccess = ApiConstants.ApiKeyAccess.fromBoolean(apiKeyAccess); + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/ManagementServerResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/ManagementServerResponse.java index a471045eb67..fc7d3b722ab 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/ManagementServerResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/ManagementServerResponse.java @@ -24,7 +24,9 @@ import org.apache.cloudstack.api.BaseResponse; import org.apache.cloudstack.api.EntityReference; import org.apache.cloudstack.management.ManagementServerHost.State; +import java.util.ArrayList; import java.util.Date; +import java.util.List; @EntityReference(value = ManagementServerHost.class) public class ManagementServerResponse extends BaseResponse { @@ -76,6 +78,10 @@ public class ManagementServerResponse extends BaseResponse { @Param(description = "the IP Address for this Management Server") private String serviceIp; + @SerializedName(ApiConstants.PEERS) + @Param(description = "the Management Server Peers") + private List peers; + public String getId() { return this.id; } @@ -171,4 +177,19 @@ public class ManagementServerResponse extends BaseResponse { public String getKernelVersion() { return kernelVersion; } + + public List getPeers() { + return peers; + } + + public void setPeers(List peers) { + this.peers = peers; + } + + public void addPeer(PeerManagementServerNodeResponse peer) { + if (peers == null) { + peers = new ArrayList<>(); + } + peers.add(peer); + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/PeerManagementServerNodeResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/PeerManagementServerNodeResponse.java new file mode 100644 index 00000000000..802294171fa --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/response/PeerManagementServerNodeResponse.java @@ -0,0 +1,100 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.response; + +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; +import org.apache.cloudstack.management.ManagementServerHost.State; + +import java.util.Date; + +public class PeerManagementServerNodeResponse extends BaseResponse { + + @SerializedName(ApiConstants.STATE) + @Param(description = "the state of the management server peer") + private State state; + + @SerializedName(ApiConstants.LAST_UPDATED) + @Param(description = "the last updated time of the management server peer state") + private Date lastUpdated; + + @SerializedName(ApiConstants.PEER_ID) + @Param(description = "the ID of the peer management server") + private String peerId; + + @SerializedName(ApiConstants.PEER_NAME) + @Param(description = "the name of the peer management server") + private String peerName; + + @SerializedName(ApiConstants.PEER_MSID) + @Param(description = "the management ID of the peer management server") + private String peerMsId; + + @SerializedName(ApiConstants.PEER_RUNID) + @Param(description = "the run ID of the peer management server") + private String peerRunId; + + @SerializedName(ApiConstants.PEER_STATE) + @Param(description = "the state of the peer management server") + private String peerState; + + @SerializedName(ApiConstants.PEER_SERVICE_IP) + @Param(description = "the IP Address for the peer Management Server") + private String peerServiceIp; + + @SerializedName(ApiConstants.PEER_SERVICE_PORT) + @Param(description = "the service port for the peer Management Server") + private String peerServicePort; + + public void setState(State state) { + this.state = state; + } + + public void setLastUpdated(Date lastUpdated) { + this.lastUpdated = lastUpdated; + } + + public void setPeerId(String peerId) { + this.peerId = peerId; + } + + public void setPeerName(String peerName) { + this.peerName = peerName; + } + + public void setPeerMsId(String peerMsId) { + this.peerMsId = peerMsId; + } + + public void setPeerRunId(String peerRunId) { + this.peerRunId = peerRunId; + } + + public void setPeerState(String peerState) { + this.peerState = peerState; + } + + public void setPeerServiceIp(String peerServiceIp) { + this.peerServiceIp = peerServiceIp; + } + + public void setPeerServicePort(String peerServicePort) { + this.peerServicePort = peerServicePort; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/response/RegisterResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/RegisterResponse.java index 5faedabfc16..dd17cc5cc8a 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/RegisterResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/RegisterResponse.java @@ -18,19 +18,24 @@ package org.apache.cloudstack.api.response; import com.google.gson.annotations.SerializedName; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseResponse; import com.cloud.serializer.Param; public class RegisterResponse extends BaseResponse { - @SerializedName("apikey") + @SerializedName(ApiConstants.API_KEY) @Param(description = "the api key of the registered user", isSensitive = true) private String apiKey; - @SerializedName("secretkey") + @SerializedName(ApiConstants.SECRET_KEY) @Param(description = "the secret key of the registered user", isSensitive = true) private String secretKey; + @SerializedName(ApiConstants.API_KEY_ACCESS) + @Param(description = "whether api key access is allowed or not", isSensitive = true) + private Boolean apiKeyAccess; + public String getApiKey() { return apiKey; } @@ -46,4 +51,8 @@ public class RegisterResponse extends BaseResponse { public void setSecretKey(String secretKey) { this.secretKey = secretKey; } + + public void setApiKeyAccess(Boolean apiKeyAccess) { + this.apiKeyAccess = apiKeyAccess; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/UserResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/UserResponse.java index 1a17f3b8698..df97a915700 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/UserResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/UserResponse.java @@ -128,6 +128,10 @@ public class UserResponse extends BaseResponse implements SetResourceIconRespons @Param(description = "true if user has two factor authentication is mandated", since = "4.18.0.0") private Boolean is2FAmandated; + @SerializedName(ApiConstants.API_KEY_ACCESS) + @Param(description = "whether api key access is Enabled, Disabled or set to Inherit (it inherits the value from the parent)", since = "4.20.1.0") + ApiConstants.ApiKeyAccess apiKeyAccess; + @Override public String getObjectId() { return this.getId(); @@ -309,4 +313,8 @@ public class UserResponse extends BaseResponse implements SetResourceIconRespons public void set2FAmandated(Boolean is2FAmandated) { this.is2FAmandated = is2FAmandated; } + + public void setApiKeyAccess(Boolean apiKeyAccess) { + this.apiKeyAccess = ApiConstants.ApiKeyAccess.fromBoolean(apiKeyAccess); + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/ZoneResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/ZoneResponse.java index 143dfad0eaf..4a5279753a1 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/ZoneResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/ZoneResponse.java @@ -157,6 +157,11 @@ public class ZoneResponse extends BaseResponseWithAnnotations implements SetReso @Param(description = "AS Number Range") private String asnRange; + @SerializedName(ApiConstants.ROUTED_MODE_ENABLED) + @Param(description = "true, if routed network/vpc is enabled", since = "4.20.1") + private boolean routedModeEnabled = false; + + public ZoneResponse() { tags = new LinkedHashSet(); } @@ -412,4 +417,12 @@ public class ZoneResponse extends BaseResponseWithAnnotations implements SetReso public String getAsnRange() { return asnRange; } + + public boolean isRoutedModeEnabled() { + return routedModeEnabled; + } + + public void setRoutedModeEnabled(boolean routedModeEnabled) { + this.routedModeEnabled = routedModeEnabled; + } } diff --git a/api/src/main/java/org/apache/cloudstack/consoleproxy/ConsoleAccessManager.java b/api/src/main/java/org/apache/cloudstack/consoleproxy/ConsoleAccessManager.java index 5bd9699b201..23b571e7fae 100644 --- a/api/src/main/java/org/apache/cloudstack/consoleproxy/ConsoleAccessManager.java +++ b/api/src/main/java/org/apache/cloudstack/consoleproxy/ConsoleAccessManager.java @@ -44,7 +44,7 @@ public interface ConsoleAccessManager extends Manager, Configurable { void removeSessions(String[] sessionUuids); - void acquireSession(String sessionUuid); + void acquireSession(String sessionUuid, String clientAddress); String genAccessTicket(String host, String port, String sid, String tag, String sessionUuid); String genAccessTicket(String host, String port, String sid, String tag, Date normalizedHashTime, String sessionUuid); diff --git a/api/src/main/java/org/apache/cloudstack/network/RoutedIpv4Manager.java b/api/src/main/java/org/apache/cloudstack/network/RoutedIpv4Manager.java index 2f704e9f47d..221a550ad63 100644 --- a/api/src/main/java/org/apache/cloudstack/network/RoutedIpv4Manager.java +++ b/api/src/main/java/org/apache/cloudstack/network/RoutedIpv4Manager.java @@ -57,6 +57,13 @@ import java.util.List; public interface RoutedIpv4Manager extends PluggableService, Configurable { + ConfigKey RoutedNetworkVpcEnabled = new ConfigKey<>(ConfigKey.CATEGORY_NETWORK, Boolean.class, + "routed.network.vpc.enabled", + "true", + "If true, the Routed network and VPC are enabled in the zone.", + true, + ConfigKey.Scope.Zone); + ConfigKey RoutedNetworkIPv4MaxCidrSize = new ConfigKey<>(ConfigKey.CATEGORY_NETWORK, Integer.class, "routed.network.ipv4.max.cidr.size", "30", "The maximum value of the cidr size for isolated networks in ROUTED mode", true, ConfigKey.Scope.Account); @@ -196,4 +203,6 @@ public interface RoutedIpv4Manager extends PluggableService, Configurable { void removeBgpPeersByAccountId(long accountId); void removeBgpPeersByDomainId(long domainId); + + Boolean isRoutedNetworkVpcEnabled(long zoneId); } diff --git a/api/src/main/java/org/apache/cloudstack/query/QueryService.java b/api/src/main/java/org/apache/cloudstack/query/QueryService.java index c93e43d9f37..88081494320 100644 --- a/api/src/main/java/org/apache/cloudstack/query/QueryService.java +++ b/api/src/main/java/org/apache/cloudstack/query/QueryService.java @@ -19,6 +19,7 @@ package org.apache.cloudstack.query; import java.util.List; import org.apache.cloudstack.affinity.AffinityGroupResponse; +import org.apache.cloudstack.api.ResponseObject; import org.apache.cloudstack.api.command.admin.domain.ListDomainsCmd; import org.apache.cloudstack.api.command.admin.host.ListHostTagsCmd; import org.apache.cloudstack.api.command.admin.host.ListHostsCmd; @@ -130,7 +131,7 @@ public interface QueryService { ConfigKey ReturnVmStatsOnVmList = new ConfigKey<>("Advanced", Boolean.class, "list.vm.default.details.stats", "true", "Determines whether VM stats should be returned when details are not explicitly specified in listVirtualMachines API request. When false, details default to [group, nics, secgrp, tmpl, servoff, diskoff, backoff, iso, volume, min, affgrp]. When true, all details are returned including 'stats'.", true, ConfigKey.Scope.Global); - ListResponse searchForUsers(ListUsersCmd cmd) throws PermissionDeniedException; + ListResponse searchForUsers(ResponseObject.ResponseView responseView, ListUsersCmd cmd) throws PermissionDeniedException; ListResponse searchForUsers(Long domainId, boolean recursive) throws PermissionDeniedException; diff --git a/client/pom.xml b/client/pom.xml index e99b8c71406..16547670c5e 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -25,7 +25,7 @@ org.apache.cloudstack cloudstack - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT diff --git a/core/pom.xml b/core/pom.xml index 83cdee8cf4f..25ac46518b3 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT diff --git a/core/src/main/java/com/cloud/agent/api/ConsoleAccessAuthenticationCommand.java b/core/src/main/java/com/cloud/agent/api/ConsoleAccessAuthenticationCommand.java index 683d4afd5b2..ac6f15ec4c3 100644 --- a/core/src/main/java/com/cloud/agent/api/ConsoleAccessAuthenticationCommand.java +++ b/core/src/main/java/com/cloud/agent/api/ConsoleAccessAuthenticationCommand.java @@ -27,6 +27,7 @@ public class ConsoleAccessAuthenticationCommand extends AgentControlCommand { private String _sid; private String _ticket; private String sessionUuid; + private String clientAddress; private boolean _isReauthenticating; @@ -35,13 +36,14 @@ public class ConsoleAccessAuthenticationCommand extends AgentControlCommand { } public ConsoleAccessAuthenticationCommand(String host, String port, String vmId, String sid, String ticket, - String sessiontkn) { + String sessiontkn, String clientAddress) { _host = host; _port = port; _vmId = vmId; _sid = sid; _ticket = ticket; sessionUuid = sessiontkn; + this.clientAddress = clientAddress; } public String getHost() { @@ -79,4 +81,12 @@ public class ConsoleAccessAuthenticationCommand extends AgentControlCommand { public void setSessionUuid(String sessionUuid) { this.sessionUuid = sessionUuid; } + + public String getClientAddress() { + return clientAddress; + } + + public void setClientAddress(String clientAddress) { + this.clientAddress = clientAddress; + } } diff --git a/core/src/main/java/com/cloud/agent/api/ConvertInstanceAnswer.java b/core/src/main/java/com/cloud/agent/api/ConvertInstanceAnswer.java index 829888570a6..174348f4f18 100644 --- a/core/src/main/java/com/cloud/agent/api/ConvertInstanceAnswer.java +++ b/core/src/main/java/com/cloud/agent/api/ConvertInstanceAnswer.java @@ -20,6 +20,8 @@ import org.apache.cloudstack.vm.UnmanagedInstanceTO; public class ConvertInstanceAnswer extends Answer { + private String temporaryConvertUuid; + public ConvertInstanceAnswer() { super(); } @@ -34,6 +36,15 @@ public class ConvertInstanceAnswer extends Answer { this.convertedInstance = convertedInstance; } + public ConvertInstanceAnswer(Command command, String temporaryConvertUuid) { + super(command, true, ""); + this.temporaryConvertUuid = temporaryConvertUuid; + } + + public String getTemporaryConvertUuid() { + return temporaryConvertUuid; + } + public UnmanagedInstanceTO getConvertedInstance() { return convertedInstance; } diff --git a/core/src/main/java/com/cloud/agent/api/ImportConvertedInstanceAnswer.java b/core/src/main/java/com/cloud/agent/api/ImportConvertedInstanceAnswer.java new file mode 100644 index 00000000000..2a8f8704e3f --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/ImportConvertedInstanceAnswer.java @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.api; + +import org.apache.cloudstack.vm.UnmanagedInstanceTO; + +public class ImportConvertedInstanceAnswer extends Answer { + + public ImportConvertedInstanceAnswer() { + super(); + } + private UnmanagedInstanceTO convertedInstance; + + public ImportConvertedInstanceAnswer(Command command, boolean success, String details) { + super(command, success, details); + } + + public ImportConvertedInstanceAnswer(Command command, UnmanagedInstanceTO convertedInstance) { + super(command, true, ""); + this.convertedInstance = convertedInstance; + } + + public UnmanagedInstanceTO getConvertedInstance() { + return convertedInstance; + } +} diff --git a/core/src/main/java/com/cloud/agent/api/ImportConvertedInstanceCommand.java b/core/src/main/java/com/cloud/agent/api/ImportConvertedInstanceCommand.java new file mode 100644 index 00000000000..9d50e852ced --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/ImportConvertedInstanceCommand.java @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.api; + +import com.cloud.agent.api.to.DataStoreTO; +import com.cloud.agent.api.to.RemoteInstanceTO; + +import java.util.List; + +public class ImportConvertedInstanceCommand extends Command { + + private RemoteInstanceTO sourceInstance; + private List destinationStoragePools; + private DataStoreTO conversionTemporaryLocation; + private String temporaryConvertUuid; + + public ImportConvertedInstanceCommand() { + } + + public ImportConvertedInstanceCommand(RemoteInstanceTO sourceInstance, + List destinationStoragePools, + DataStoreTO conversionTemporaryLocation, String temporaryConvertUuid) { + this.sourceInstance = sourceInstance; + this.destinationStoragePools = destinationStoragePools; + this.conversionTemporaryLocation = conversionTemporaryLocation; + this.temporaryConvertUuid = temporaryConvertUuid; + } + + public RemoteInstanceTO getSourceInstance() { + return sourceInstance; + } + + public List getDestinationStoragePools() { + return destinationStoragePools; + } + + public DataStoreTO getConversionTemporaryLocation() { + return conversionTemporaryLocation; + } + + public String getTemporaryConvertUuid() { + return temporaryConvertUuid; + } + + @Override + public boolean executeInSequence() { + return false; + } +} diff --git a/core/src/main/java/com/cloud/agent/resource/virtualnetwork/facade/SetPortForwardingRulesConfigItem.java b/core/src/main/java/com/cloud/agent/resource/virtualnetwork/facade/SetPortForwardingRulesConfigItem.java index c9d0e74e2e4..4daef64ed8a 100644 --- a/core/src/main/java/com/cloud/agent/resource/virtualnetwork/facade/SetPortForwardingRulesConfigItem.java +++ b/core/src/main/java/com/cloud/agent/resource/virtualnetwork/facade/SetPortForwardingRulesConfigItem.java @@ -41,7 +41,7 @@ public class SetPortForwardingRulesConfigItem extends AbstractConfigItemFacade { for (final PortForwardingRuleTO rule : command.getRules()) { final ForwardingRule fwdRule = new ForwardingRule(rule.revoked(), rule.getProtocol().toLowerCase(), rule.getSrcIp(), rule.getStringSrcPortRange(), rule.getDstIp(), - rule.getStringDstPortRange()); + rule.getStringDstPortRange(), rule.getSourceCidrListAsString()); rules.add(fwdRule); } diff --git a/core/src/main/java/com/cloud/agent/resource/virtualnetwork/model/ForwardingRule.java b/core/src/main/java/com/cloud/agent/resource/virtualnetwork/model/ForwardingRule.java index cf3e43d1c01..1dc1cc855c4 100644 --- a/core/src/main/java/com/cloud/agent/resource/virtualnetwork/model/ForwardingRule.java +++ b/core/src/main/java/com/cloud/agent/resource/virtualnetwork/model/ForwardingRule.java @@ -26,18 +26,21 @@ public class ForwardingRule { private String sourcePortRange; private String destinationIpAddress; private String destinationPortRange; + private String sourceCidrList; public ForwardingRule() { // Empty constructor for (de)serialization } - public ForwardingRule(boolean revoke, String protocol, String sourceIpAddress, String sourcePortRange, String destinationIpAddress, String destinationPortRange) { + public ForwardingRule(boolean revoke, String protocol, String sourceIpAddress, String sourcePortRange, String destinationIpAddress, String destinationPortRange, + String sourceCidrList) { this.revoke = revoke; this.protocol = protocol; this.sourceIpAddress = sourceIpAddress; this.sourcePortRange = sourcePortRange; this.destinationIpAddress = destinationIpAddress; this.destinationPortRange = destinationPortRange; + this.sourceCidrList = sourceCidrList; } public boolean isRevoke() { @@ -88,4 +91,8 @@ public class ForwardingRule { this.destinationPortRange = destinationPortRange; } + public String getSourceCidrList() { + return sourceCidrList; + } + } diff --git a/debian/changelog b/debian/changelog index cbc4fcfdb2d..d17676ef268 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,12 @@ -cloudstack (4.20.0.0-SNAPSHOT) unstable; urgency=low +cloudstack (4.21.0.0-SNAPSHOT) unstable; urgency=low - * Update the version to 4.20.0.0-SNAPSHOT + * Update the version to 4.21.0.0-SNAPSHOT + + -- the Apache CloudStack project Tue, 19 Nov 2024 08:54:07 -0300 + +cloudstack (4.21.0.0-SNAPSHOT-SNAPSHOT) unstable; urgency=low + + * Update the version to 4.21.0.0-SNAPSHOT-SNAPSHOT -- the Apache CloudStack project Mon, 29 Jan 2024 10:21:52 +0530 diff --git a/developer/pom.xml b/developer/pom.xml index b44446ec0a5..c573c03d008 100644 --- a/developer/pom.xml +++ b/developer/pom.xml @@ -25,7 +25,7 @@ org.apache.cloudstack cloudstack - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT diff --git a/engine/api/pom.xml b/engine/api/pom.xml index 1112e6eff8b..25aaa1365df 100644 --- a/engine/api/pom.xml +++ b/engine/api/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloud-engine - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../pom.xml diff --git a/engine/components-api/pom.xml b/engine/components-api/pom.xml index b06b644d67f..0b30decffda 100644 --- a/engine/components-api/pom.xml +++ b/engine/components-api/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloud-engine - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../pom.xml diff --git a/engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java b/engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java index 1c3edad886b..e1bb10f5d26 100644 --- a/engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java +++ b/engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java @@ -40,6 +40,7 @@ public interface CapacityManager { static final String StorageCapacityDisableThresholdCK = "pool.storage.capacity.disablethreshold"; static final String StorageOverprovisioningFactorCK = "storage.overprovisioning.factor"; static final String StorageAllocatedCapacityDisableThresholdCK = "pool.storage.allocated.capacity.disablethreshold"; + static final String StorageAllocatedCapacityDisableThresholdForVolumeResizeCK = "pool.storage.allocated.resize.capacity.disablethreshold"; static final ConfigKey CpuOverprovisioningFactor = new ConfigKey<>( @@ -118,6 +119,17 @@ public interface CapacityManager { "Percentage (as a value between 0 and 1) of secondary storage capacity threshold.", true); + static final ConfigKey StorageAllocatedCapacityDisableThresholdForVolumeSize = + new ConfigKey<>( + ConfigKey.CATEGORY_ALERT, + Double.class, + StorageAllocatedCapacityDisableThresholdForVolumeResizeCK, + "0.90", + "Percentage (as a value between 0 and 1) of allocated storage utilization above which allocators will disable using the pool for volume resize. " + + "This is applicable only when volume.resize.allowed.beyond.allocation is set to true.", + true, + ConfigKey.Scope.Zone); + public boolean releaseVmCapacity(VirtualMachine vm, boolean moveFromReserved, boolean moveToReservered, Long hostId); void allocateVmCapacity(VirtualMachine vm, boolean fromLastHost); diff --git a/engine/components-api/src/main/java/com/cloud/network/vpc/VpcManager.java b/engine/components-api/src/main/java/com/cloud/network/vpc/VpcManager.java index a340f49c13f..e7f02e62045 100644 --- a/engine/components-api/src/main/java/com/cloud/network/vpc/VpcManager.java +++ b/engine/components-api/src/main/java/com/cloud/network/vpc/VpcManager.java @@ -37,8 +37,26 @@ import com.cloud.network.PhysicalNetwork; import com.cloud.network.addr.PublicIp; import com.cloud.offering.NetworkOffering; import com.cloud.user.Account; +import org.apache.cloudstack.framework.config.ConfigKey; public interface VpcManager { + ConfigKey VpcTierNamePrepend = new ConfigKey<>(Boolean.class, + "vpc.tier.name.prepend", + ConfigKey.CATEGORY_NETWORK, + "false", + "Whether to prepend the VPC name to the VPC tier network name", + true, + ConfigKey.Scope.Global, + null); + ConfigKey VpcTierNamePrependDelimiter = new ConfigKey<>(String.class, + "vpc.tier.name.prepend.delimiter", + ConfigKey.CATEGORY_NETWORK, + " ", + "Delimiter string to use between the VPC and the VPC tier name", + true, + ConfigKey.Scope.Global, + null); + /** * Returns all the Guest networks that are part of VPC * diff --git a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java index c3909bc56b0..b5153668899 100644 --- a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java +++ b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java @@ -209,6 +209,11 @@ public interface StorageManager extends StorageService { ConfigKey HEURISTICS_SCRIPT_TIMEOUT = new ConfigKey<>("Advanced", Long.class, "heuristics.script.timeout", "3000", "The maximum runtime, in milliseconds, to execute the heuristic rule; if it is reached, a timeout will happen.", true); + ConfigKey AllowVolumeReSizeBeyondAllocation = new ConfigKey("Advanced", Boolean.class, "volume.resize.allowed.beyond.allocation", "false", + "Determines whether volume size can exceed the pool capacity allocation disable threshold (pool.storage.allocated.capacity.disablethreshold) " + + "when resize a volume upto resize capacity disable threshold (pool.storage.allocated.resize.capacity.disablethreshold)", + true, ConfigKey.Scope.Zone); + /** * should we execute in sequence not involving any storages? * @return tru if commands should execute in sequence diff --git a/engine/components-api/src/main/java/com/cloud/vm/snapshot/VMSnapshotManager.java b/engine/components-api/src/main/java/com/cloud/vm/snapshot/VMSnapshotManager.java index 82456004cc3..a01d4ee5cae 100644 --- a/engine/components-api/src/main/java/com/cloud/vm/snapshot/VMSnapshotManager.java +++ b/engine/components-api/src/main/java/com/cloud/vm/snapshot/VMSnapshotManager.java @@ -31,7 +31,7 @@ public interface VMSnapshotManager extends VMSnapshotService, Manager { static final ConfigKey VMSnapshotExpireInterval = new ConfigKey("Advanced", Integer.class, "vmsnapshot.expire.interval", "-1", "VM Snapshot expire interval in hours", true, ConfigKey.Scope.Account); - public static final int VMSNAPSHOTMAX = 10; + ConfigKey VMSnapshotMax = new ConfigKey("Advanced", Integer.class, "vmsnapshot.max", "10", "Maximum vm snapshots for a single vm", true, ConfigKey.Scope.Global); /** * Delete all VM snapshots belonging to one VM diff --git a/engine/orchestration/pom.xml b/engine/orchestration/pom.xml index e4953fc0cbe..bf8ab14c952 100755 --- a/engine/orchestration/pom.xml +++ b/engine/orchestration/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloud-engine - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../pom.xml diff --git a/engine/pom.xml b/engine/pom.xml index 5e52544aeca..1df7d84284f 100644 --- a/engine/pom.xml +++ b/engine/pom.xml @@ -25,7 +25,7 @@ org.apache.cloudstack cloudstack - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../pom.xml diff --git a/engine/schema/pom.xml b/engine/schema/pom.xml index 82120ae70cc..8238da84835 100644 --- a/engine/schema/pom.xml +++ b/engine/schema/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloud-engine - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../pom.xml diff --git a/engine/schema/src/main/java/com/cloud/network/dao/FirewallRulesCidrsDao.java b/engine/schema/src/main/java/com/cloud/network/dao/FirewallRulesCidrsDao.java index 55c454860ef..df5b6b5d647 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/FirewallRulesCidrsDao.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/FirewallRulesCidrsDao.java @@ -29,4 +29,6 @@ public interface FirewallRulesCidrsDao extends GenericDao listByFirewallRuleId(long firewallRuleId); + + void updateSourceCidrsForRule(Long firewallRuleId, List sourceCidrList); } diff --git a/engine/schema/src/main/java/com/cloud/network/dao/FirewallRulesCidrsDaoImpl.java b/engine/schema/src/main/java/com/cloud/network/dao/FirewallRulesCidrsDaoImpl.java index fdd1e0ec43a..6279289bdfe 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/FirewallRulesCidrsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/FirewallRulesCidrsDaoImpl.java @@ -20,6 +20,7 @@ import java.util.ArrayList; import java.util.List; +import org.apache.commons.collections4.CollectionUtils; import org.springframework.stereotype.Component; import com.cloud.utils.db.DB; @@ -45,7 +46,7 @@ public class FirewallRulesCidrsDaoImpl extends GenericDaoBase results = search(sc, null); - List cidrs = new ArrayList(results.size()); + List cidrs = new ArrayList<>(results.size()); for (FirewallRulesCidrsVO result : results) { cidrs.add(result.getCidr()); } @@ -63,9 +64,27 @@ public class FirewallRulesCidrsDaoImpl extends GenericDaoBase sourceCidrList) { + TransactionLegacy txn = TransactionLegacy.currentTxn(); + txn.start(); + + SearchCriteria sc = CidrsSearch.create(); + sc.setParameters("firewallRuleId", firewallRuleId); + remove(sc); + + persist(firewallRuleId, sourceCidrList); + + txn.commit(); + } + @Override @DB public void persist(long firewallRuleId, List sourceCidrs) { + if (CollectionUtils.isEmpty(sourceCidrs)) { + return; + } + TransactionLegacy txn = TransactionLegacy.currentTxn(); txn.start(); diff --git a/engine/schema/src/main/java/com/cloud/network/dao/FirewallRulesDaoImpl.java b/engine/schema/src/main/java/com/cloud/network/dao/FirewallRulesDaoImpl.java index 1698e0ed2da..c8bd7e2147e 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/FirewallRulesDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/FirewallRulesDaoImpl.java @@ -251,9 +251,6 @@ public class FirewallRulesDaoImpl extends GenericDaoBase i } public void saveSourceCidrs(FirewallRuleVO firewallRule, List cidrList) { - if (cidrList == null) { - return; - } _firewallRulesCidrsDao.persist(firewallRule.getId(), cidrList); } diff --git a/engine/schema/src/main/java/com/cloud/network/rules/PortForwardingRuleVO.java b/engine/schema/src/main/java/com/cloud/network/rules/PortForwardingRuleVO.java index e1a698881f3..576e2f8172e 100644 --- a/engine/schema/src/main/java/com/cloud/network/rules/PortForwardingRuleVO.java +++ b/engine/schema/src/main/java/com/cloud/network/rules/PortForwardingRuleVO.java @@ -25,6 +25,7 @@ import javax.persistence.EnumType; import javax.persistence.Enumerated; import javax.persistence.PrimaryKeyJoinColumn; import javax.persistence.Table; +import javax.persistence.Transient; import com.cloud.utils.net.Ip; @@ -47,21 +48,30 @@ public class PortForwardingRuleVO extends FirewallRuleVO implements PortForwardi @Column(name = "instance_id") private long virtualMachineId; + @Transient + List sourceCidrs; + public PortForwardingRuleVO() { } public PortForwardingRuleVO(String xId, long srcIpId, int srcPortStart, int srcPortEnd, Ip dstIp, int dstPortStart, int dstPortEnd, String protocol, long networkId, - long accountId, long domainId, long instanceId) { - super(xId, srcIpId, srcPortStart, srcPortEnd, protocol, networkId, accountId, domainId, Purpose.PortForwarding, null, null, null, null, null); + long accountId, long domainId, long instanceId, List sourceCidrs) { + super(xId, srcIpId, srcPortStart, srcPortEnd, protocol, networkId, accountId, domainId, Purpose.PortForwarding, sourceCidrs, null, null, null, null); this.destinationIpAddress = dstIp; this.virtualMachineId = instanceId; this.destinationPortStart = dstPortStart; this.destinationPortEnd = dstPortEnd; + this.sourceCidrs = sourceCidrs; } - public PortForwardingRuleVO(String xId, long srcIpId, int srcPort, Ip dstIp, int dstPort, String protocol, List sourceCidrs, long networkId, long accountId, - long domainId, long instanceId) { - this(xId, srcIpId, srcPort, srcPort, dstIp, dstPort, dstPort, protocol.toLowerCase(), networkId, accountId, domainId, instanceId); + public PortForwardingRuleVO(String xId, long srcIpId, int srcPortStart, int srcPortEnd, Ip dstIp, int dstPortStart, int dstPortEnd, String protocol, long networkId, + long accountId, long domainId, long instanceId) { + this(xId, srcIpId, srcPortStart, srcPortEnd, dstIp, dstPortStart, dstPortEnd, protocol.toLowerCase(), networkId, accountId, domainId, instanceId, null); + } + + public PortForwardingRuleVO(String xId, long srcIpId, int srcPort, Ip dstIp, int dstPort, String protocol, long networkId, long accountId, + long domainId, long instanceId) { + this(xId, srcIpId, srcPort, srcPort, dstIp, dstPort, dstPort, protocol.toLowerCase(), networkId, accountId, domainId, instanceId, null); } @Override @@ -106,4 +116,13 @@ public class PortForwardingRuleVO extends FirewallRuleVO implements PortForwardi return null; } + public void setSourceCidrList(List sourceCidrs) { + this.sourceCidrs = sourceCidrs; + } + + @Override + public List getSourceCidrList() { + return sourceCidrs; + } + } diff --git a/engine/schema/src/main/java/com/cloud/network/rules/dao/PortForwardingRulesDaoImpl.java b/engine/schema/src/main/java/com/cloud/network/rules/dao/PortForwardingRulesDaoImpl.java index 3a404b3f2df..1b3df06e1a2 100644 --- a/engine/schema/src/main/java/com/cloud/network/rules/dao/PortForwardingRulesDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/network/rules/dao/PortForwardingRulesDaoImpl.java @@ -31,6 +31,9 @@ import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.SearchCriteria.Op; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallback; +import com.cloud.utils.db.TransactionLegacy; @Component public class PortForwardingRulesDaoImpl extends GenericDaoBase implements PortForwardingRulesDao { @@ -42,7 +45,7 @@ public class PortForwardingRulesDaoImpl extends GenericDaoBase ActiveRulesSearchByAccount; @Inject - protected FirewallRulesCidrsDao _portForwardingRulesCidrsDao; + protected FirewallRulesCidrsDao portForwardingRulesCidrsDao; protected PortForwardingRulesDaoImpl() { super(); @@ -183,4 +186,43 @@ public class PortForwardingRulesDaoImpl extends GenericDaoBase) transactionStatus -> { + PortForwardingRuleVO dbPfRule = super.persist(portForwardingRule); + + portForwardingRulesCidrsDao.persist(portForwardingRule.getId(), portForwardingRule.getSourceCidrList()); + List cidrList = portForwardingRulesCidrsDao.getSourceCidrs(portForwardingRule.getId()); + portForwardingRule.setSourceCidrList(cidrList); + + return dbPfRule; + }); + + } + + @Override + public boolean update(Long id, PortForwardingRuleVO entity) { + TransactionLegacy txn = TransactionLegacy.currentTxn(); + txn.start(); + + boolean success = super.update(id, entity); + if (!success) { + return false; + } + + portForwardingRulesCidrsDao.updateSourceCidrsForRule(entity.getId(), entity.getSourceCidrList()); + txn.commit(); + + return true; + } + + @Override + public PortForwardingRuleVO findById(Long id) { + PortForwardingRuleVO rule = super.findById(id); + + List sourceCidrList = portForwardingRulesCidrsDao.getSourceCidrs(id); + rule.setSourceCidrList(sourceCidrList); + + return rule; + } } diff --git a/engine/schema/src/main/java/com/cloud/projects/dao/ProjectAccountDao.java b/engine/schema/src/main/java/com/cloud/projects/dao/ProjectAccountDao.java index 730182a1cb5..f4b2f646002 100644 --- a/engine/schema/src/main/java/com/cloud/projects/dao/ProjectAccountDao.java +++ b/engine/schema/src/main/java/com/cloud/projects/dao/ProjectAccountDao.java @@ -47,6 +47,8 @@ public interface ProjectAccountDao extends GenericDao { void removeAccountFromProjects(long accountId); + void removeUserFromProjects(long userId); + boolean canUserModifyProject(long projectId, long accountId, long userId); List listUsersOrAccountsByRole(long id); diff --git a/engine/schema/src/main/java/com/cloud/projects/dao/ProjectAccountDaoImpl.java b/engine/schema/src/main/java/com/cloud/projects/dao/ProjectAccountDaoImpl.java index 8947cc600b3..b6eb6d44cea 100644 --- a/engine/schema/src/main/java/com/cloud/projects/dao/ProjectAccountDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/projects/dao/ProjectAccountDaoImpl.java @@ -192,6 +192,17 @@ public class ProjectAccountDaoImpl extends GenericDaoBase sc = AllFieldsSearch.create(); + sc.setParameters("userId", userId); + + int removedCount = remove(sc); + if (removedCount > 0) { + logger.debug(String.format("Removed user [%s] from %s project(s).", userId, removedCount)); + } + } + @Override public boolean canUserModifyProject(long projectId, long accountId, long userId) { SearchCriteria sc = AllFieldsSearch.create(); diff --git a/engine/schema/src/main/java/com/cloud/storage/VolumeVO.java b/engine/schema/src/main/java/com/cloud/storage/VolumeVO.java index c105acf40b8..ea57ef91237 100644 --- a/engine/schema/src/main/java/com/cloud/storage/VolumeVO.java +++ b/engine/schema/src/main/java/com/cloud/storage/VolumeVO.java @@ -48,31 +48,34 @@ public class VolumeVO implements Volume { @TableGenerator(name = "volume_sq", table = "sequence", pkColumnName = "name", valueColumnName = "value", pkColumnValue = "volume_seq", allocationSize = 1) @GeneratedValue(strategy = GenerationType.TABLE) @Column(name = "id") - long id; + private long id; + + @Column(name = "last_id") + private long lastId; @Column(name = "name") - String name; + private String name; @Column(name = "pool_id") - Long poolId; + private Long poolId; @Column(name = "last_pool_id") - Long lastPoolId; + private Long lastPoolId; @Column(name = "account_id") - long accountId; + private long accountId; @Column(name = "domain_id") - long domainId; + private long domainId; @Column(name = "instance_id") - Long instanceId = null; + private Long instanceId = null; @Column(name = "device_id") - Long deviceId = null; + private Long deviceId = null; @Column(name = "size") - Long size; + private Long size; @Column(name = "min_iops") private Long minIops; @@ -81,50 +84,50 @@ public class VolumeVO implements Volume { private Long maxIops; @Column(name = "folder") - String folder; + private String folder; @Column(name = "path") - String path; + private String path; @Column(name = "pod_id") - Long podId; + private Long podId; @Column(name = "created") - Date created; + private Date created; @Column(name = "attached") @Temporal(value = TemporalType.TIMESTAMP) - Date attached; + private Date attached; @Column(name = "data_center_id") - long dataCenterId; + private long dataCenterId; @Column(name = "host_ip") - String hostip; + private String hostIp; @Column(name = "disk_offering_id") - long diskOfferingId; + private long diskOfferingId; @Column(name = "template_id") - Long templateId; + private Long templateId; @Column(name = "first_snapshot_backup_uuid") - String firstSnapshotBackupUuid; + private String firstSnapshotBackupUuid; @Column(name = "volume_type") @Enumerated(EnumType.STRING) - Type volumeType = Volume.Type.UNKNOWN; + private Type volumeType = Volume.Type.UNKNOWN; @Column(name = "pool_type") @Convert(converter = StoragePoolTypeConverter.class) - StoragePoolType poolType; + private StoragePoolType poolType; @Column(name = GenericDao.REMOVED_COLUMN) - Date removed; + private Date removed; @Column(name = "updated") @Temporal(value = TemporalType.TIMESTAMP) - Date updated; + private Date updated; @Column(name = "update_count", updatable = true, nullable = false) protected long updatedCount; // This field should be updated everytime the @@ -133,17 +136,17 @@ public class VolumeVO implements Volume { // dao code. @Column(name = "recreatable") - boolean recreatable; + private boolean recreatable; @Column(name = "state") @Enumerated(value = EnumType.STRING) private State state; @Column(name = "chain_info", length = 65535) - String chainInfo; + private String chainInfo; @Column(name = "uuid") - String uuid; + private String uuid; @Column(name = "format") private Storage.ImageFormat format; @@ -168,7 +171,7 @@ public class VolumeVO implements Volume { @Transient // @Column(name="reservation") - String reservationId; + private String reservationId; @Column(name = "hv_ss_reserve") private Integer hypervisorSnapshotReserve; @@ -428,11 +431,11 @@ public class VolumeVO implements Volume { } public String getHostIp() { - return hostip; + return hostIp; } public void setHostIp(String hostip) { - this.hostip = hostip; + this.hostIp = hostip; } public void setPodId(Long podId) { @@ -690,4 +693,12 @@ public class VolumeVO implements Volume { public void setDeleteProtection(boolean deleteProtection) { this.deleteProtection = deleteProtection; } + + public long getLastId() { + return lastId; + } + + public void setLastId(long lastId) { + this.lastId = lastId; + } } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java index cb219007325..1e3b3a7e5ec 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java @@ -88,6 +88,8 @@ import com.cloud.upgrade.dao.Upgrade41800to41810; import com.cloud.upgrade.dao.Upgrade41810to41900; import com.cloud.upgrade.dao.Upgrade41900to41910; import com.cloud.upgrade.dao.Upgrade41910to42000; +import com.cloud.upgrade.dao.Upgrade42000to42010; +import com.cloud.upgrade.dao.Upgrade42010to42100; import com.cloud.upgrade.dao.Upgrade420to421; import com.cloud.upgrade.dao.Upgrade421to430; import com.cloud.upgrade.dao.Upgrade430to440; @@ -230,6 +232,8 @@ public class DatabaseUpgradeChecker implements SystemIntegrityChecker { .next("4.18.1.0", new Upgrade41810to41900()) .next("4.19.0.0", new Upgrade41900to41910()) .next("4.19.1.0", new Upgrade41910to42000()) + .next("4.20.0.0", new Upgrade42000to42010()) + .next("4.20.1.0", new Upgrade42010to42100()) .build(); } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41910to41920.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41910to41920.java new file mode 100644 index 00000000000..6215021473e --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41910to41920.java @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.upgrade.dao; + +import com.cloud.utils.exception.CloudRuntimeException; + +import java.io.InputStream; +import java.sql.Connection; + +public class Upgrade41910to41920 implements DbUpgrade { + + @Override + public String[] getUpgradableVersionRange() { + return new String[]{"4.19.1.0", "4.19.2.0"}; + } + + @Override + public String getUpgradedVersion() { + return "4.19.2.0"; + } + + @Override + public boolean supportsRollingUpgrade() { + return false; + } + + @Override + public InputStream[] getPrepareScripts() { + final String scriptFile = "META-INF/db/schema-41910to41920.sql"; + final InputStream script = Thread.currentThread().getContextClassLoader().getResourceAsStream(scriptFile); + if (script == null) { + throw new CloudRuntimeException("Unable to find " + scriptFile); + } + + return new InputStream[]{script}; + } + + @Override + public void performDataMigration(Connection conn) { + } + + @Override + public InputStream[] getCleanupScripts() { + final String scriptFile = "META-INF/db/schema-41910to41920-cleanup.sql"; + final InputStream script = Thread.currentThread().getContextClassLoader().getResourceAsStream(scriptFile); + if (script == null) { + throw new CloudRuntimeException("Unable to find " + scriptFile); + } + + return new InputStream[]{script}; + } +} diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42000to42010.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42000to42010.java new file mode 100644 index 00000000000..197ca1cb34c --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42000to42010.java @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.upgrade.dao; + +import java.io.InputStream; +import java.sql.Connection; + +import com.cloud.upgrade.SystemVmTemplateRegistration; +import com.cloud.utils.exception.CloudRuntimeException; + +public class Upgrade42000to42010 extends DbUpgradeAbstractImpl implements DbUpgrade, DbUpgradeSystemVmTemplate { + private SystemVmTemplateRegistration systemVmTemplateRegistration; + + @Override + public String[] getUpgradableVersionRange() { + return new String[] {"4.20.0.0", "4.20.1.0"}; + } + + @Override + public String getUpgradedVersion() { + return "4.20.1.0"; + } + + @Override + public boolean supportsRollingUpgrade() { + return false; + } + + @Override + public InputStream[] getPrepareScripts() { + final String scriptFile = "META-INF/db/schema-42000to42010.sql"; + final InputStream script = Thread.currentThread().getContextClassLoader().getResourceAsStream(scriptFile); + if (script == null) { + throw new CloudRuntimeException("Unable to find " + scriptFile); + } + + return new InputStream[] {script}; + } + + @Override + public void performDataMigration(Connection conn) { + } + + @Override + public InputStream[] getCleanupScripts() { + final String scriptFile = "META-INF/db/schema-42000to42010-cleanup.sql"; + final InputStream script = Thread.currentThread().getContextClassLoader().getResourceAsStream(scriptFile); + if (script == null) { + throw new CloudRuntimeException("Unable to find " + scriptFile); + } + + return new InputStream[] {script}; + } + + private void initSystemVmTemplateRegistration() { + systemVmTemplateRegistration = new SystemVmTemplateRegistration(""); + } + + @Override + public void updateSystemVmTemplates(Connection conn) { + logger.debug("Updating System Vm template IDs"); + initSystemVmTemplateRegistration(); + try { + systemVmTemplateRegistration.updateSystemVmTemplates(conn); + } catch (Exception e) { + throw new CloudRuntimeException("Failed to find / register SystemVM template(s)"); + } + } +} diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42010to42100.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42010to42100.java new file mode 100644 index 00000000000..06a68ec3d8b --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42010to42100.java @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.upgrade.dao; + +import com.cloud.upgrade.SystemVmTemplateRegistration; +import com.cloud.utils.exception.CloudRuntimeException; + +import java.io.InputStream; +import java.sql.Connection; + +public class Upgrade42010to42100 extends DbUpgradeAbstractImpl implements DbUpgrade, DbUpgradeSystemVmTemplate { + private SystemVmTemplateRegistration systemVmTemplateRegistration; + + @Override + public String[] getUpgradableVersionRange() { + return new String[] {"4.20.1.0", "4.21.0.0"}; + } + + @Override + public String getUpgradedVersion() { + return "4.21.0.0"; + } + + @Override + public boolean supportsRollingUpgrade() { + return false; + } + + @Override + public InputStream[] getPrepareScripts() { + final String scriptFile = "META-INF/db/schema-42010to42100.sql"; + final InputStream script = Thread.currentThread().getContextClassLoader().getResourceAsStream(scriptFile); + if (script == null) { + throw new CloudRuntimeException("Unable to find " + scriptFile); + } + + return new InputStream[] {script}; + } + + @Override + public void performDataMigration(Connection conn) { + } + + @Override + public InputStream[] getCleanupScripts() { + final String scriptFile = "META-INF/db/schema-42010to42100-cleanup.sql"; + final InputStream script = Thread.currentThread().getContextClassLoader().getResourceAsStream(scriptFile); + if (script == null) { + throw new CloudRuntimeException("Unable to find " + scriptFile); + } + + return new InputStream[] {script}; + } + + private void initSystemVmTemplateRegistration() { + systemVmTemplateRegistration = new SystemVmTemplateRegistration(""); + } + + @Override + public void updateSystemVmTemplates(Connection conn) { + logger.debug("Updating System Vm template IDs"); + initSystemVmTemplateRegistration(); + try { + systemVmTemplateRegistration.updateSystemVmTemplates(conn); + } catch (Exception e) { + throw new CloudRuntimeException("Failed to find / register SystemVM template(s)"); + } + } +} diff --git a/engine/schema/src/main/java/com/cloud/user/AccountVO.java b/engine/schema/src/main/java/com/cloud/user/AccountVO.java index f04b2bafbde..74a538565d7 100644 --- a/engine/schema/src/main/java/com/cloud/user/AccountVO.java +++ b/engine/schema/src/main/java/com/cloud/user/AccountVO.java @@ -77,6 +77,9 @@ public class AccountVO implements Account { @Column(name = "default") boolean isDefault; + @Column(name = "api_key_access") + private Boolean apiKeyAccess; + public AccountVO() { uuid = UUID.randomUUID().toString(); } @@ -229,4 +232,14 @@ public class AccountVO implements Account { public String reflectionToString() { return ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, "id", "uuid", "accountName", "domainId"); } + + @Override + public void setApiKeyAccess(Boolean apiKeyAccess) { + this.apiKeyAccess = apiKeyAccess; + } + + @Override + public Boolean getApiKeyAccess() { + return apiKeyAccess; + } } diff --git a/engine/schema/src/main/java/com/cloud/user/UserVO.java b/engine/schema/src/main/java/com/cloud/user/UserVO.java index 69970bf2d2c..7dac26429ac 100644 --- a/engine/schema/src/main/java/com/cloud/user/UserVO.java +++ b/engine/schema/src/main/java/com/cloud/user/UserVO.java @@ -115,6 +115,9 @@ public class UserVO implements User, Identity, InternalIdentity { @Column(name = "key_for_2fa") private String keyFor2fa; + @Column(name = "api_key_access") + private Boolean apiKeyAccess; + public UserVO() { this.uuid = UUID.randomUUID().toString(); } @@ -350,4 +353,13 @@ public class UserVO implements User, Identity, InternalIdentity { this.user2faProvider = user2faProvider; } + @Override + public void setApiKeyAccess(Boolean apiKeyAccess) { + this.apiKeyAccess = apiKeyAccess; + } + + @Override + public Boolean getApiKeyAccess() { + return apiKeyAccess; + } } diff --git a/engine/schema/src/main/java/com/cloud/user/dao/AccountDaoImpl.java b/engine/schema/src/main/java/com/cloud/user/dao/AccountDaoImpl.java index eed5572a0b2..f9ef5c40eba 100644 --- a/engine/schema/src/main/java/com/cloud/user/dao/AccountDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/user/dao/AccountDaoImpl.java @@ -41,8 +41,8 @@ import java.util.List; @Component public class AccountDaoImpl extends GenericDaoBase implements AccountDao { - private static final String FIND_USER_ACCOUNT_BY_API_KEY = "SELECT u.id, u.username, u.account_id, u.secret_key, u.state, " - + "a.id, a.account_name, a.type, a.role_id, a.domain_id, a.state " + "FROM `cloud`.`user` u, `cloud`.`account` a " + private static final String FIND_USER_ACCOUNT_BY_API_KEY = "SELECT u.id, u.username, u.account_id, u.secret_key, u.state, u.api_key_access, " + + "a.id, a.account_name, a.type, a.role_id, a.domain_id, a.state, a.api_key_access " + "FROM `cloud`.`user` u, `cloud`.`account` a " + "WHERE u.account_id = a.id AND u.api_key = ? and u.removed IS NULL"; protected final SearchBuilder AllFieldsSearch; @@ -148,13 +148,25 @@ public class AccountDaoImpl extends GenericDaoBase implements A u.setAccountId(rs.getLong(3)); u.setSecretKey(DBEncryptionUtil.decrypt(rs.getString(4))); u.setState(State.getValueOf(rs.getString(5))); + boolean apiKeyAccess = rs.getBoolean(6); + if (rs.wasNull()) { + u.setApiKeyAccess(null); + } else { + u.setApiKeyAccess(apiKeyAccess); + } - AccountVO a = new AccountVO(rs.getLong(6)); - a.setAccountName(rs.getString(7)); - a.setType(Account.Type.getFromValue(rs.getInt(8))); - a.setRoleId(rs.getLong(9)); - a.setDomainId(rs.getLong(10)); - a.setState(State.getValueOf(rs.getString(11))); + AccountVO a = new AccountVO(rs.getLong(7)); + a.setAccountName(rs.getString(8)); + a.setType(Account.Type.getFromValue(rs.getInt(9))); + a.setRoleId(rs.getLong(10)); + a.setDomainId(rs.getLong(11)); + a.setState(State.getValueOf(rs.getString(12))); + apiKeyAccess = rs.getBoolean(13); + if (rs.wasNull()) { + a.setApiKeyAccess(null); + } else { + a.setApiKeyAccess(apiKeyAccess); + } userAcctPair = new Pair(u, a); } diff --git a/engine/schema/src/main/java/com/cloud/vm/ConsoleSessionVO.java b/engine/schema/src/main/java/com/cloud/vm/ConsoleSessionVO.java index 81a11241e4b..ef777be2de9 100644 --- a/engine/schema/src/main/java/com/cloud/vm/ConsoleSessionVO.java +++ b/engine/schema/src/main/java/com/cloud/vm/ConsoleSessionVO.java @@ -64,6 +64,12 @@ public class ConsoleSessionVO { @Column(name = "removed") private Date removed; + @Column(name = "console_endpoint_creator_address") + private String consoleEndpointCreatorAddress; + + @Column(name = "client_address") + private String clientAddress; + public long getId() { return id; } @@ -135,4 +141,20 @@ public class ConsoleSessionVO { public void setAcquired(Date acquired) { this.acquired = acquired; } + + public String getConsoleEndpointCreatorAddress() { + return consoleEndpointCreatorAddress; + } + + public void setConsoleEndpointCreatorAddress(String consoleEndpointCreatorAddress) { + this.consoleEndpointCreatorAddress = consoleEndpointCreatorAddress; + } + + public String getClientAddress() { + return clientAddress; + } + + public void setClientAddress(String clientAddress) { + this.clientAddress = clientAddress; + } } diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/ConsoleSessionDao.java b/engine/schema/src/main/java/com/cloud/vm/dao/ConsoleSessionDao.java index 79158dd13b2..95ced889b3d 100644 --- a/engine/schema/src/main/java/com/cloud/vm/dao/ConsoleSessionDao.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/ConsoleSessionDao.java @@ -33,7 +33,7 @@ public interface ConsoleSessionDao extends GenericDao { int expungeSessionsOlderThanDate(Date date); - void acquireSession(String sessionUuid); + void acquireSession(String sessionUuid, String clientAddress); int expungeByVmList(List vmIds, Long batchSize); } diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/ConsoleSessionDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/ConsoleSessionDaoImpl.java index 48709674451..3d117894670 100644 --- a/engine/schema/src/main/java/com/cloud/vm/dao/ConsoleSessionDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/ConsoleSessionDaoImpl.java @@ -62,9 +62,10 @@ public class ConsoleSessionDaoImpl extends GenericDaoBase + diff --git a/engine/schema/src/main/resources/META-INF/db/procedures/cloud.idempotent_add_foreign_key.sql b/engine/schema/src/main/resources/META-INF/db/procedures/cloud.idempotent_add_foreign_key.sql new file mode 100644 index 00000000000..754c02acb93 --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/procedures/cloud.idempotent_add_foreign_key.sql @@ -0,0 +1,28 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +DROP PROCEDURE IF EXISTS `cloud`.`IDEMPOTENT_ADD_FOREIGN_KEY`; + +CREATE PROCEDURE `cloud`.`IDEMPOTENT_ADD_FOREIGN_KEY` ( + IN in_table_name VARCHAR(200) + , IN in_key_name VARCHAR(200) + , IN in_foreign_key VARCHAR(200) + , IN in_references VARCHAR(1000) +) +BEGIN + + DECLARE CONTINUE HANDLER FOR 1061 BEGIN END; SET @ddl = CONCAT_WS(' ', 'ALTER TABLE ', in_table_name, ' ADD CONSTRAINT ', in_key_name, ' FOREIGN KEY ', in_foreign_key, ' REFERENCES ', in_references, ' ON DELETE CASCADE'); PREPARE stmt FROM @ddl; EXECUTE stmt; DEALLOCATE PREPARE stmt; END; diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41910to41920-cleanup.sql b/engine/schema/src/main/resources/META-INF/db/schema-41910to41920-cleanup.sql new file mode 100644 index 00000000000..cb317c69b79 --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/schema-41910to41920-cleanup.sql @@ -0,0 +1,23 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +--; +-- Schema upgrade cleanup from 4.19.1.0 to 4.19.2.0 +--; + +-- Delete `project_account` entries for users that were removed +DELETE FROM `cloud`.`project_account` WHERE `user_id` IN (SELECT `id` FROM `cloud`.`user` WHERE `removed`); diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41910to41920.sql b/engine/schema/src/main/resources/META-INF/db/schema-41910to41920.sql new file mode 100644 index 00000000000..2ce8ea99bd1 --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/schema-41910to41920.sql @@ -0,0 +1,23 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +--; +-- Schema upgrade from 4.19.1.0 to 4.19.2.0 +--; + +-- Add last_id to the volumes table +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.volumes', 'last_id', 'bigint(20) unsigned DEFAULT NULL'); diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42000to42010-cleanup.sql b/engine/schema/src/main/resources/META-INF/db/schema-42000to42010-cleanup.sql new file mode 100644 index 00000000000..d187b6fa043 --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/schema-42000to42010-cleanup.sql @@ -0,0 +1,20 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +--; +-- Schema upgrade cleanup from 4.20.0.0 to 4.20.1.0 +--; diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42000to42010.sql b/engine/schema/src/main/resources/META-INF/db/schema-42000to42010.sql new file mode 100644 index 00000000000..aef99dd0c7f --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/schema-42000to42010.sql @@ -0,0 +1,34 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +--; +-- Schema upgrade from 4.20.0.0 to 4.20.1.0 +--; + +-- Add column api_key_access to user and account tables +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.user', 'api_key_access', 'boolean DEFAULT NULL COMMENT "is api key access allowed for the user" AFTER `secret_key`'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.account', 'api_key_access', 'boolean DEFAULT NULL COMMENT "is api key access allowed for the account" '); + +-- Modify index for mshost_peer +DELETE FROM `cloud`.`mshost_peer`; +CALL `cloud`.`IDEMPOTENT_DROP_FOREIGN_KEY`('cloud.mshost_peer','fk_mshost_peer__owner_mshost'); +CALL `cloud`.`IDEMPOTENT_DROP_INDEX`('i_mshost_peer__owner_peer_runid','mshost_peer'); +CALL `cloud`.`IDEMPOTENT_ADD_UNIQUE_KEY`('cloud.mshost_peer', 'i_mshost_peer__owner_peer', '(owner_mshost, peer_mshost)'); +CALL `cloud`.`IDEMPOTENT_ADD_FOREIGN_KEY`('cloud.mshost_peer', 'fk_mshost_peer__owner_mshost', '(owner_mshost)', '`mshost`(`id`)'); + +-- Add last_id to the volumes table +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.volumes', 'last_id', 'bigint(20) unsigned DEFAULT NULL'); diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42010to42100-cleanup.sql b/engine/schema/src/main/resources/META-INF/db/schema-42010to42100-cleanup.sql new file mode 100644 index 00000000000..5f257f2965b --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/schema-42010to42100-cleanup.sql @@ -0,0 +1,20 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +--; +-- Schema upgrade cleanup from 4.20.1.0 to 4.21.0.0 +--; diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql b/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql new file mode 100644 index 00000000000..91223bab798 --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql @@ -0,0 +1,26 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +--; +-- Schema upgrade from 4.20.1.0 to 4.21.0.0 +--; + +-- Add console_endpoint_creator_address column to cloud.console_session table +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.console_session', 'console_endpoint_creator_address', 'VARCHAR(45)'); + +-- Add client_address column to cloud.console_session table +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.console_session', 'client_address', 'VARCHAR(45)'); diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.account_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.account_view.sql index 87546a9d118..dc64380fb57 100644 --- a/engine/schema/src/main/resources/META-INF/db/views/cloud.account_view.sql +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.account_view.sql @@ -31,6 +31,7 @@ select `account`.`cleanup_needed` AS `cleanup_needed`, `account`.`network_domain` AS `network_domain` , `account`.`default` AS `default`, + `account`.`api_key_access` AS `api_key_access`, `domain`.`id` AS `domain_id`, `domain`.`uuid` AS `domain_uuid`, `domain`.`name` AS `domain_name`, diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.mshost_peer_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.mshost_peer_view.sql new file mode 100644 index 00000000000..5f741449d85 --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.mshost_peer_view.sql @@ -0,0 +1,44 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + + +DROP VIEW IF EXISTS `cloud`.`mshost_peer_view`; + +CREATE VIEW `cloud`.`mshost_peer_view` AS +SELECT + `mshost_peer`.`id` AS `id`, + `mshost_peer`.`peer_state` AS `peer_state`, + `mshost_peer`.`last_update` AS `last_update`, + `owner_mshost`.`id` AS `owner_mshost_id`, + `owner_mshost`.`msid` AS `owner_mshost_msid`, + `owner_mshost`.`runid` AS `owner_mshost_runid`, + `owner_mshost`.`name` AS `owner_mshost_name`, + `owner_mshost`.`uuid` AS `owner_mshost_uuid`, + `owner_mshost`.`state` AS `owner_mshost_state`, + `owner_mshost`.`service_ip` AS `owner_mshost_service_ip`, + `owner_mshost`.`service_port` AS `owner_mshost_service_port`, + `peer_mshost`.`id` AS `peer_mshost_id`, + `peer_mshost`.`msid` AS `peer_mshost_msid`, + `peer_mshost`.`runid` AS `peer_mshost_runid`, + `peer_mshost`.`name` AS `peer_mshost_name`, + `peer_mshost`.`uuid` AS `peer_mshost_uuid`, + `peer_mshost`.`state` AS `peer_mshost_state`, + `peer_mshost`.`service_ip` AS `peer_mshost_service_ip`, + `peer_mshost`.`service_port` AS `peer_mshost_service_port` +FROM `cloud`.`mshost_peer` +LEFT JOIN `cloud`.`mshost` AS owner_mshost on `mshost_peer`.`owner_mshost` = `owner_mshost`.`id` +LEFT JOIN `cloud`.`mshost` AS peer_mshost on `mshost_peer`.`peer_mshost` = `peer_mshost`.`id`; diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.user_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.user_view.sql index 7eedc03712b..340cfa9055f 100644 --- a/engine/schema/src/main/resources/META-INF/db/views/cloud.user_view.sql +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.user_view.sql @@ -39,6 +39,7 @@ select user.incorrect_login_attempts, user.source, user.default, + user.api_key_access, account.id account_id, account.uuid account_uuid, account.account_name account_name, diff --git a/engine/service/pom.xml b/engine/service/pom.xml index 34221e1001d..83179d2db65 100644 --- a/engine/service/pom.xml +++ b/engine/service/pom.xml @@ -22,7 +22,7 @@ org.apache.cloudstack cloud-engine - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT cloud-engine-service war diff --git a/engine/storage/cache/pom.xml b/engine/storage/cache/pom.xml index a1b7aff7afd..60822f761bc 100644 --- a/engine/storage/cache/pom.xml +++ b/engine/storage/cache/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloud-engine - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../../pom.xml diff --git a/engine/storage/configdrive/pom.xml b/engine/storage/configdrive/pom.xml index b14acf10138..e66f38cd6c3 100644 --- a/engine/storage/configdrive/pom.xml +++ b/engine/storage/configdrive/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloud-engine - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../../pom.xml diff --git a/engine/storage/datamotion/pom.xml b/engine/storage/datamotion/pom.xml index 5620ca8ef70..ff1859a99d8 100644 --- a/engine/storage/datamotion/pom.xml +++ b/engine/storage/datamotion/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloud-engine - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../../pom.xml diff --git a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java index 9755dcbb0df..3456731ef1c 100644 --- a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java +++ b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java @@ -2293,6 +2293,8 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy { if (success) { VolumeVO volumeVO = _volumeDao.findById(destVolumeInfo.getId()); volumeVO.setFormat(ImageFormat.QCOW2); + volumeVO.setLastId(srcVolumeInfo.getId()); + _volumeDao.update(volumeVO.getId(), volumeVO); _volumeService.copyPoliciesBetweenVolumesAndDestroySourceVolumeAfterMigration(Event.OperationSuccessed, null, srcVolumeInfo, destVolumeInfo, false); diff --git a/engine/storage/image/pom.xml b/engine/storage/image/pom.xml index 278b3672b2f..c5023e24534 100644 --- a/engine/storage/image/pom.xml +++ b/engine/storage/image/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloud-engine - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../../pom.xml diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java index 5e21f37f4d5..abc955c2e49 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java @@ -533,6 +533,11 @@ public class TemplateServiceImpl implements TemplateService { logger.info("Skip downloading template " + tmplt.getUniqueName() + " since no url is specified."); continue; } + // if this is private template, skip sync to a new image store + if (isSkipTemplateStoreDownload(tmplt, zoneId)) { + logger.info("Skip sync downloading private template " + tmplt.getUniqueName() + " to a new image store"); + continue; + } // if this is a region store, and there is already an DOWNLOADED entry there without install_path information, which // means that this is a duplicate entry from migration of previous NFS to staging. diff --git a/engine/storage/integration-test/pom.xml b/engine/storage/integration-test/pom.xml index a5bc225f4f6..ff0b64f5eaf 100644 --- a/engine/storage/integration-test/pom.xml +++ b/engine/storage/integration-test/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloud-engine - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../../pom.xml diff --git a/engine/storage/object/pom.xml b/engine/storage/object/pom.xml index 7159a646fbb..97859ec68db 100644 --- a/engine/storage/object/pom.xml +++ b/engine/storage/object/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloud-engine - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../../pom.xml diff --git a/engine/storage/pom.xml b/engine/storage/pom.xml index e16e88e235d..7588b61eb39 100644 --- a/engine/storage/pom.xml +++ b/engine/storage/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloud-engine - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../pom.xml diff --git a/engine/storage/snapshot/pom.xml b/engine/storage/snapshot/pom.xml index f29b43d8de0..db70d53647a 100644 --- a/engine/storage/snapshot/pom.xml +++ b/engine/storage/snapshot/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloud-engine - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../../pom.xml diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/DefaultVMSnapshotStrategy.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/DefaultVMSnapshotStrategy.java index 1d3788a0301..09f569e6f19 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/DefaultVMSnapshotStrategy.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/DefaultVMSnapshotStrategy.java @@ -419,7 +419,7 @@ public class DefaultVMSnapshotStrategy extends ManagerBase implements VMSnapshot if (answer != null && answer.getDetails() != null) errMsg = errMsg + " due to " + answer.getDetails(); logger.error(errMsg); - throw new CloudRuntimeException(errMsg); + throw new CloudRuntimeException(String.format("Unable to revert VM %s to snapshot %s.", userVm.getInstanceName(), vmSnapshotVO.getName())); } } catch (OperationTimedoutException e) { logger.debug("Failed to revert vm snapshot", e); diff --git a/engine/storage/volume/pom.xml b/engine/storage/volume/pom.xml index a00c3314126..7709a7c2c5b 100644 --- a/engine/storage/volume/pom.xml +++ b/engine/storage/volume/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloud-engine - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../../pom.xml diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java index 9a3319f79a3..3ca1d9201db 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java @@ -1720,6 +1720,7 @@ public class VolumeServiceImpl implements VolumeService { newVol.setPassphraseId(volume.getPassphraseId()); newVol.setEncryptFormat(volume.getEncryptFormat()); } + newVol.setLastId(volume.getId()); return volDao.persist(newVol); } diff --git a/engine/userdata/cloud-init/pom.xml b/engine/userdata/cloud-init/pom.xml index d4396ba382a..70e2d70f94b 100644 --- a/engine/userdata/cloud-init/pom.xml +++ b/engine/userdata/cloud-init/pom.xml @@ -23,7 +23,7 @@ cloud-engine org.apache.cloudstack - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../../pom.xml diff --git a/engine/userdata/pom.xml b/engine/userdata/pom.xml index 038aa18f290..a97a308c1ab 100644 --- a/engine/userdata/pom.xml +++ b/engine/userdata/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloud-engine - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../pom.xml diff --git a/framework/agent-lb/pom.xml b/framework/agent-lb/pom.xml index 50e0bd47b90..14b692d6a45 100644 --- a/framework/agent-lb/pom.xml +++ b/framework/agent-lb/pom.xml @@ -24,7 +24,7 @@ cloudstack-framework org.apache.cloudstack - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../pom.xml diff --git a/framework/ca/pom.xml b/framework/ca/pom.xml index d82389cd008..691e9e3a9da 100644 --- a/framework/ca/pom.xml +++ b/framework/ca/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-framework - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../pom.xml diff --git a/framework/cluster/pom.xml b/framework/cluster/pom.xml index ef511584ae6..b2e89704c89 100644 --- a/framework/cluster/pom.xml +++ b/framework/cluster/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-framework - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../pom.xml diff --git a/framework/cluster/src/main/java/com/cloud/cluster/ClusterManager.java b/framework/cluster/src/main/java/com/cloud/cluster/ClusterManager.java index 54f575830e4..7fdaa6f7f77 100644 --- a/framework/cluster/src/main/java/com/cloud/cluster/ClusterManager.java +++ b/framework/cluster/src/main/java/com/cloud/cluster/ClusterManager.java @@ -27,9 +27,9 @@ import com.cloud.utils.component.Manager; public interface ClusterManager extends Manager { static final String ALERT_SUBJECT = "cluster-alert"; final ConfigKey HeartbeatInterval = new ConfigKey(Integer.class, "cluster.heartbeat.interval", "management-server", "1500", - "Interval to check for the heart beat between management server nodes", false); + "Interval (in milliseconds) to check for the heart beat between management server nodes", false); final ConfigKey HeartbeatThreshold = new ConfigKey(Integer.class, "cluster.heartbeat.threshold", "management-server", "150000", - "Threshold before self-fence the management server", true); + "Threshold (in milliseconds) before self-fence the management server. The threshold should be larger than management.server.stats.interval", true); /** * Adds a new packet to the incoming queue. diff --git a/framework/cluster/src/main/java/com/cloud/cluster/ClusterManagerImpl.java b/framework/cluster/src/main/java/com/cloud/cluster/ClusterManagerImpl.java index 32fdf782696..0ec566a4194 100644 --- a/framework/cluster/src/main/java/com/cloud/cluster/ClusterManagerImpl.java +++ b/framework/cluster/src/main/java/com/cloud/cluster/ClusterManagerImpl.java @@ -758,21 +758,16 @@ public class ClusterManagerImpl extends ManagerBase implements ClusterManager, C } switch (msg.getMessageType()) { - case nodeAdded: { - final List l = msg.getNodes(); - if (l != null && l.size() > 0) { - for (final ManagementServerHostVO mshost : l) { - _mshostPeerDao.updatePeerInfo(_mshostId, mshost.getId(), mshost.getRunid(), ManagementServerHost.State.Up); - } - } - } + case nodeAdded: break; case nodeRemoved: { final List l = msg.getNodes(); if (l != null && l.size() > 0) { for (final ManagementServerHostVO mshost : l) { - _mshostPeerDao.updatePeerInfo(_mshostId, mshost.getId(), mshost.getRunid(), ManagementServerHost.State.Down); + if (mshost.getId() != _mshostId) { + _mshostPeerDao.updatePeerInfo(_mshostId, mshost.getId(), mshost.getRunid(), ManagementServerHost.State.Down); + } } } } @@ -823,8 +818,9 @@ public class ClusterManagerImpl extends ManagerBase implements ClusterManager, C final List downHostList = new ArrayList(); for (final ManagementServerHostVO host : inactiveList) { - if (!pingManagementNode(host)) { - logger.warn("Management node " + host.getId() + " is detected inactive by timestamp and also not pingable"); + // Check if peer state is Up in the period + if (!_mshostPeerDao.isPeerUpState(_mshostId, host.getId(), new Date(cutTime.getTime() - HeartbeatThreshold.value()))) { + logger.warn("Management node " + host.getId() + " is detected inactive by timestamp and did not send node status to this node"); downHostList.add(host); } } @@ -898,6 +894,44 @@ public class ClusterManagerImpl extends ManagerBase implements ClusterManager, C final Profiler profilerInvalidatedNodeList = new Profiler(); profilerInvalidatedNodeList.start(); + processInvalidatedNodes(invalidatedNodeList); + profilerInvalidatedNodeList.stop(); + + final Profiler profilerRemovedList = new Profiler(); + profilerRemovedList.start(); + processRemovedNodes(cutTime, removedNodeList); + profilerRemovedList.stop(); + + final Profiler profilerNewList = new Profiler(); + profilerNewList.start(); + processNewNodes(cutTime, currentList); + profilerNewList.stop(); + + final Profiler profilerInactiveList = new Profiler(); + profilerInactiveList.start(); + processInactiveNodes(cutTime); + profilerInactiveList.stop(); + + profiler.stop(); + + logger.debug(String.format("Peer scan is finished. profiler: %s , profilerQueryActiveList: %s, " + + ", profilerSyncClusterInfo: %s, profilerInvalidatedNodeList: %s, profilerRemovedList: %s," + + ", profilerNewList: %s, profilerInactiveList: %s", + profiler, profilerQueryActiveList, profilerSyncClusterInfo, profilerInvalidatedNodeList, profilerRemovedList, + profilerNewList, profilerInactiveList)); + + if (profiler.getDurationInMillis() >= HeartbeatInterval.value()) { + if (logger.isDebugEnabled()) { + logger.debug(String.format("Peer scan takes too long to finish. profiler: %s , profilerQueryActiveList: %s, " + + ", profilerSyncClusterInfo: %s, profilerInvalidatedNodeList: %s, profilerRemovedList: %s," + + ", profilerNewList: %s, profilerInactiveList: %s", + profiler, profilerQueryActiveList, profilerSyncClusterInfo, profilerInvalidatedNodeList, profilerRemovedList, + profilerNewList, profilerInactiveList)); + } + } + } + + private void processInvalidatedNodes(List invalidatedNodeList) { // process invalidated node list if (invalidatedNodeList.size() > 0) { for (final ManagementServerHostVO mshost : invalidatedNodeList) { @@ -911,16 +945,16 @@ public class ClusterManagerImpl extends ManagerBase implements ClusterManager, C queueNotification(new ClusterManagerMessage(ClusterManagerMessage.MessageType.nodeRemoved, invalidatedNodeList)); } - profilerInvalidatedNodeList.stop(); + } - final Profiler profilerRemovedList = new Profiler(); - profilerRemovedList.start(); + private void processRemovedNodes(Date cutTime, List removedNodeList) { // process removed node list final Iterator it = removedNodeList.iterator(); while (it.hasNext()) { final ManagementServerHostVO mshost = it.next(); - if (!pingManagementNode(mshost)) { - logger.warn("Management node " + mshost.getId() + " is detected inactive by timestamp and also not pingable"); + // Check if peer state is Up in the period + if (!_mshostPeerDao.isPeerUpState(_mshostId, mshost.getId(), new Date(cutTime.getTime() - HeartbeatThreshold.value()))) { + logger.warn("Management node " + mshost.getId() + " is detected inactive by timestamp and did not send node status to this node"); _activePeers.remove(mshost.getId()); try { JmxUtil.unregisterMBean("ClusterManager", "Node " + mshost.getId()); @@ -928,7 +962,7 @@ public class ClusterManagerImpl extends ManagerBase implements ClusterManager, C logger.warn("Unable to deregiester cluster node from JMX monitoring due to exception " + e.toString()); } } else { - logger.info("Management node " + mshost.getId() + " is detected inactive by timestamp but is pingable"); + logger.info("Management node " + mshost.getId() + " is detected inactive by timestamp but sent node status to this node"); it.remove(); } } @@ -936,8 +970,9 @@ public class ClusterManagerImpl extends ManagerBase implements ClusterManager, C if (removedNodeList.size() > 0) { queueNotification(new ClusterManagerMessage(ClusterManagerMessage.MessageType.nodeRemoved, removedNodeList)); } - profilerRemovedList.stop(); + } + private void processNewNodes(Date cutTime, List currentList) { final List newNodeList = new ArrayList(); for (final ManagementServerHostVO mshost : currentList) { if (!_activePeers.containsKey(mshost.getId())) { @@ -959,18 +994,31 @@ public class ClusterManagerImpl extends ManagerBase implements ClusterManager, C if (newNodeList.size() > 0) { queueNotification(new ClusterManagerMessage(ClusterManagerMessage.MessageType.nodeAdded, newNodeList)); } + } - profiler.stop(); - - if (profiler.getDurationInMillis() >= HeartbeatInterval.value()) { - if (logger.isDebugEnabled()) { - logger.debug("Peer scan takes too long to finish. profiler: " + profiler.toString() + ", profilerQueryActiveList: " + - profilerQueryActiveList.toString() + ", profilerSyncClusterInfo: " + profilerSyncClusterInfo.toString() + ", profilerInvalidatedNodeList: " + - profilerInvalidatedNodeList.toString() + ", profilerRemovedList: " + profilerRemovedList.toString()); + private void processInactiveNodes(Date cutTime) { + final List inactiveList = _mshostDao.getInactiveList(new Date(cutTime.getTime() - HeartbeatThreshold.value())); + if (inactiveList.size() > 0) { + if (logger.isInfoEnabled()) { + logger.info(String.format("Found %s inactive management server node based on timestamp", inactiveList.size())); } + for (final ManagementServerHostVO host : inactiveList) { + logger.info(String.format("management server node msid: %s, name: %s, service ip: %s, version: %s", + host.getMsid(), host.getName(), host.getServiceIP(), host.getVersion())); + // Check if any peer state is Up in the period + if (ManagementServerHost.State.Up.equals(host.getState()) && + !_mshostPeerDao.isPeerUpState(host.getId(), new Date(cutTime.getTime() - HeartbeatThreshold.value()))) { + logger.warn("Management node " + host.getId() + " is detected inactive by timestamp and did not send node status to all other nodes"); + host.setState(ManagementServerHost.State.Down); + _mshostDao.update(host.getId(), host); + } + } + } else { + logger.info("No inactive management server node found"); } } + private static ManagementServerHostVO getInListById(final Long id, final List l) { for (final ManagementServerHostVO mshost : l) { if (mshost.getId() == id) { diff --git a/framework/cluster/src/main/java/com/cloud/cluster/ManagementServerHostPeerJoinVO.java b/framework/cluster/src/main/java/com/cloud/cluster/ManagementServerHostPeerJoinVO.java new file mode 100644 index 00000000000..673db160b3c --- /dev/null +++ b/framework/cluster/src/main/java/com/cloud/cluster/ManagementServerHostPeerJoinVO.java @@ -0,0 +1,177 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.cluster; + +import java.util.Date; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; +import javax.persistence.Temporal; +import javax.persistence.TemporalType; + +import org.apache.cloudstack.management.ManagementServerHost; + +@Entity +@Table(name = "mshost_peer_view") +public class ManagementServerHostPeerJoinVO { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + private long id; + + @Column(name = "peer_state") + @Enumerated(value = EnumType.STRING) + private ManagementServerHost.State peerState; + + @Temporal(TemporalType.TIMESTAMP) + @Column(name = "last_update") + private Date lastUpdateTime; + + @Column(name = "owner_mshost_id") + private long ownerMshostId; + + @Column(name = "owner_mshost_msid") + private long ownerMshostMsId; + + @Column(name = "owner_mshost_runid") + private long ownerMshostRunId; + + @Column(name = "owner_mshost_name") + private String ownerMshostName; + + @Column(name = "owner_mshost_uuid") + private String ownerMshostUuid; + + @Column(name = "owner_mshost_state") + private String ownerMshostState; + + @Column(name = "owner_mshost_service_ip") + private String ownerMshostServiceIp; + + @Column(name = "owner_mshost_service_port") + private Integer ownerMshostServicePort; + + @Column(name = "peer_mshost_id") + private long peerMshostId; + + @Column(name = "peer_mshost_msid") + private long peerMshostMsId; + + @Column(name = "peer_mshost_runid") + private long peerMshostRunId; + + @Column(name = "peer_mshost_name") + private String peerMshostName; + + @Column(name = "peer_mshost_uuid") + private String peerMshostUuid; + + @Column(name = "peer_mshost_state") + private String peerMshostState; + + @Column(name = "peer_mshost_service_ip") + private String peerMshostServiceIp; + + @Column(name = "peer_mshost_service_port") + private Integer peerMshostServicePort; + + public ManagementServerHostPeerJoinVO() { + } + + public long getId() { + return id; + } + + public ManagementServerHost.State getPeerState() { + return peerState; + } + + public Date getLastUpdateTime() { + return lastUpdateTime; + } + + public long getOwnerMshostId() { + return ownerMshostId; + } + + public long getOwnerMshostMsId() { + return ownerMshostMsId; + } + + public long getOwnerMshostRunId() { + return ownerMshostRunId; + } + + public String getOwnerMshostName() { + return ownerMshostName; + } + + public String getOwnerMshostUuid() { + return ownerMshostUuid; + } + + public String getOwnerMshostState() { + return ownerMshostState; + } + + public String getOwnerMshostServiceIp() { + return ownerMshostServiceIp; + } + + public Integer getOwnerMshostServicePort() { + return ownerMshostServicePort; + } + + public long getPeerMshostId() { + return peerMshostId; + } + + public long getPeerMshostMsId() { + return peerMshostMsId; + } + + public long getPeerMshostRunId() { + return peerMshostRunId; + } + + public String getPeerMshostName() { + return peerMshostName; + } + + public String getPeerMshostUuid() { + return peerMshostUuid; + } + + public String getPeerMshostState() { + return peerMshostState; + } + + public String getPeerMshostServiceIp() { + return peerMshostServiceIp; + } + + public Integer getPeerMshostServicePort() { + return peerMshostServicePort; + } +} diff --git a/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDaoImpl.java b/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDaoImpl.java index 7b69889c853..27b6d52f61b 100644 --- a/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDaoImpl.java +++ b/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDaoImpl.java @@ -130,7 +130,7 @@ public class ManagementServerHostDaoImpl extends GenericDaoBase { void clearPeerInfo(long ownerMshost); void updatePeerInfo(long ownerMshost, long peerMshost, long peerRunid, ManagementServerHost.State peerState); - int countStateSeenInPeers(long mshost, long runid, ManagementServerHost.State state); + int countStateSeenInPeers(long peerMshost, long runid, ManagementServerHost.State state); + + boolean isPeerUpState(long peerMshost, Date cutTime); + + boolean isPeerUpState(long ownerMshost, long peerMshost, Date cutTime); + } diff --git a/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostPeerDaoImpl.java b/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostPeerDaoImpl.java index 827be4fe299..ec69f5817ac 100644 --- a/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostPeerDaoImpl.java +++ b/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostPeerDaoImpl.java @@ -16,10 +16,10 @@ // under the License. package com.cloud.cluster.dao; +import java.util.Date; import java.util.List; - import org.apache.cloudstack.management.ManagementServerHost; import com.cloud.cluster.ManagementServerHostPeerVO; import com.cloud.utils.db.DB; @@ -33,10 +33,12 @@ public class ManagementServerHostPeerDaoImpl extends GenericDaoBase ClearPeerSearch; private final SearchBuilder FindForUpdateSearch; private final SearchBuilder CountSearch; + private final SearchBuilder ActiveSearch; public ManagementServerHostPeerDaoImpl() { ClearPeerSearch = createSearchBuilder(); ClearPeerSearch.and("ownerMshost", ClearPeerSearch.entity().getOwnerMshost(), SearchCriteria.Op.EQ); + ClearPeerSearch.or("peerMshost", ClearPeerSearch.entity().getPeerMshost(), SearchCriteria.Op.EQ); ClearPeerSearch.done(); FindForUpdateSearch = createSearchBuilder(); @@ -50,6 +52,13 @@ public class ManagementServerHostPeerDaoImpl extends GenericDaoBase sc = ClearPeerSearch.create(); sc.setParameters("ownerMshost", ownerMshost); + sc.setParameters("peerMshost", ownerMshost); expunge(sc); } @@ -71,11 +81,12 @@ public class ManagementServerHostPeerDaoImpl extends GenericDaoBase sc = FindForUpdateSearch.create(); sc.setParameters("ownerMshost", ownerMshost); sc.setParameters("peerMshost", peerMshost); - sc.setParameters("peerRunid", peerRunid); List l = listBy(sc); if (l.size() == 1) { ManagementServerHostPeerVO peer = l.get(0); + peer.setPeerRunid(peerRunid); peer.setPeerState(peerState); + peer.setLastUpdateTime(new Date()); update(peer.getId(), peer); } else { ManagementServerHostPeerVO peer = new ManagementServerHostPeerVO(ownerMshost, peerMshost, peerRunid, peerState); @@ -90,13 +101,36 @@ public class ManagementServerHostPeerDaoImpl extends GenericDaoBase sc = CountSearch.create(); - sc.setParameters("peerMshost", mshost); + sc.setParameters("peerMshost", peerMshost); sc.setParameters("peerRunid", runid); sc.setParameters("peerState", state); List l = listBy(sc); return l.size(); } + + @Override + @DB + public boolean isPeerUpState(long peerMshost, Date cutTime) { + SearchCriteria sc = ActiveSearch.create(); + sc.setParameters("peerMshost", peerMshost); + sc.setParameters("peerState", ManagementServerHost.State.Up); + sc.setParameters("lastUpdateTime", cutTime); + + return listBy(sc).size() > 0; + } + + @Override + @DB + public boolean isPeerUpState(long ownerMshost, long peerMshost, Date cutTime) { + SearchCriteria sc = ActiveSearch.create(); + sc.setParameters("ownerMshost", ownerMshost); + sc.setParameters("peerMshost", peerMshost); + sc.setParameters("peerState", ManagementServerHost.State.Up); + sc.setParameters("lastUpdateTime", cutTime); + + return listBy(sc).size() > 0; + } } diff --git a/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostPeerJoinDao.java b/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostPeerJoinDao.java new file mode 100644 index 00000000000..46f87b6484c --- /dev/null +++ b/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostPeerJoinDao.java @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.cluster.dao; + +import com.cloud.cluster.ManagementServerHostPeerJoinVO; +import com.cloud.utils.db.GenericDao; + +import java.util.List; + +public interface ManagementServerHostPeerJoinDao extends GenericDao { + + List listByOwnerMshostId(long ownerMshostId); +} diff --git a/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostPeerJoinDaoImpl.java b/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostPeerJoinDaoImpl.java new file mode 100644 index 00000000000..16a17863d04 --- /dev/null +++ b/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostPeerJoinDaoImpl.java @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.cluster.dao; + +import java.util.List; + +import com.cloud.cluster.ManagementServerHostPeerJoinVO; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; + +public class ManagementServerHostPeerJoinDaoImpl extends GenericDaoBase implements ManagementServerHostPeerJoinDao { + + private final SearchBuilder AllFieldSearch; + + public ManagementServerHostPeerJoinDaoImpl() { + AllFieldSearch = createSearchBuilder(); + AllFieldSearch.and("ownerMshostId", AllFieldSearch.entity().getOwnerMshostId(), SearchCriteria.Op.EQ); + AllFieldSearch.done(); + } + + @Override + public List listByOwnerMshostId(long ownerMshostId) { + SearchCriteria sc = AllFieldSearch.create(); + sc.setParameters("ownerMshostId", ownerMshostId); + return listBy(sc); + } +} diff --git a/framework/config/pom.xml b/framework/config/pom.xml index fc3b14642f1..403429380b6 100644 --- a/framework/config/pom.xml +++ b/framework/config/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-framework - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../pom.xml diff --git a/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java b/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java index 36a8050754c..00cf56345c8 100644 --- a/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java +++ b/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java @@ -34,6 +34,7 @@ public class ConfigKey { public static final String CATEGORY_ADVANCED = "Advanced"; public static final String CATEGORY_ALERT = "Alert"; public static final String CATEGORY_NETWORK = "Network"; + public static final String CATEGORY_SYSTEM = "System"; public enum Scope { Global, Zone, Cluster, StoragePool, Account, ManagementServer, ImageStore, Domain diff --git a/framework/db/pom.xml b/framework/db/pom.xml index 586d72f34f3..db6421a4717 100644 --- a/framework/db/pom.xml +++ b/framework/db/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-framework - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../pom.xml diff --git a/framework/db/src/main/java/com/cloud/utils/crypt/EncryptionSecretKeyChanger.java b/framework/db/src/main/java/com/cloud/utils/crypt/EncryptionSecretKeyChanger.java index 961c537d0da..e96b0a00894 100644 --- a/framework/db/src/main/java/com/cloud/utils/crypt/EncryptionSecretKeyChanger.java +++ b/framework/db/src/main/java/com/cloud/utils/crypt/EncryptionSecretKeyChanger.java @@ -656,7 +656,7 @@ public class EncryptionSecretKeyChanger { String sqlTemplateDeployAsIsDetails = "SELECT template_deploy_as_is_details.value " + "FROM template_deploy_as_is_details JOIN vm_instance " + "WHERE template_deploy_as_is_details.template_id = vm_instance.vm_template_id " + - "vm_instance.id = %s AND template_deploy_as_is_details.name = '%s' LIMIT 1"; + "AND vm_instance.id = %s AND template_deploy_as_is_details.name = '%s' LIMIT 1"; try (PreparedStatement selectPstmt = conn.prepareStatement("SELECT id, vm_id, name, value FROM user_vm_deploy_as_is_details"); ResultSet rs = selectPstmt.executeQuery(); PreparedStatement updatePstmt = conn.prepareStatement("UPDATE user_vm_deploy_as_is_details SET value=? WHERE id=?") diff --git a/framework/direct-download/pom.xml b/framework/direct-download/pom.xml index 1915377f222..eeb6049285a 100644 --- a/framework/direct-download/pom.xml +++ b/framework/direct-download/pom.xml @@ -32,7 +32,7 @@ cloudstack-framework org.apache.cloudstack - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../pom.xml diff --git a/framework/events/pom.xml b/framework/events/pom.xml index 3f457920cc9..20e2a02fd45 100644 --- a/framework/events/pom.xml +++ b/framework/events/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-framework - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../pom.xml diff --git a/framework/ipc/pom.xml b/framework/ipc/pom.xml index 3c03ed04e28..66812b1051c 100644 --- a/framework/ipc/pom.xml +++ b/framework/ipc/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-framework - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../pom.xml diff --git a/framework/jobs/pom.xml b/framework/jobs/pom.xml index a82f514635f..a7c8699d0c7 100644 --- a/framework/jobs/pom.xml +++ b/framework/jobs/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-framework - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../pom.xml diff --git a/framework/managed-context/pom.xml b/framework/managed-context/pom.xml index bc7fa17940b..32c07bc701d 100644 --- a/framework/managed-context/pom.xml +++ b/framework/managed-context/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../../pom.xml diff --git a/framework/pom.xml b/framework/pom.xml index 79b1036eae7..77a2710c335 100644 --- a/framework/pom.xml +++ b/framework/pom.xml @@ -25,7 +25,7 @@ org.apache.cloudstack cloudstack - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT diff --git a/framework/quota/pom.xml b/framework/quota/pom.xml index 2e608d7a248..a44ac18f2f9 100644 --- a/framework/quota/pom.xml +++ b/framework/quota/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-framework - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../pom.xml diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/QuotaManagerImpl.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/QuotaManagerImpl.java index 226a47bb7df..c9254814f46 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/QuotaManagerImpl.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/QuotaManagerImpl.java @@ -157,7 +157,7 @@ public class QuotaManagerImpl extends ManagerBase implements QuotaManager { .map(quotaUsageVO -> new Pair<>(quotaUsageVO.getStartDate(), quotaUsageVO.getEndDate())) .collect(Collectors.toCollection(LinkedHashSet::new)); - logger.info(String.format("Processing quota balance for account[{}] between [{}] and [{}].", accountToString, startDate, lastQuotaUsageEndDate)); + logger.info("Processing quota balance for account [{}] between [{}] and [{}].", accountToString, startDate, lastQuotaUsageEndDate); long accountId = accountVo.getAccountId(); long domainId = accountVo.getDomainId(); diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariables.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariables.java index b27bf589c16..6dab6604e91 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariables.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariables.java @@ -17,6 +17,8 @@ package org.apache.cloudstack.quota.activationrule.presetvariables; +import java.util.List; + public class PresetVariables { @PresetVariableDefinition(description = "Account owner of the resource.") @@ -37,6 +39,9 @@ public class PresetVariables { @PresetVariableDefinition(description = "Zone where the resource is.") private GenericPresetVariable zone; + @PresetVariableDefinition(description = "A list containing the tariffs ordered by the field 'position'.") + private List lastTariffs; + public Account getAccount() { return account; } @@ -84,4 +89,12 @@ public class PresetVariables { public void setZone(GenericPresetVariable zone) { this.zone = zone; } + + public List getLastTariffs() { + return lastTariffs; + } + + public void setLastTariffs(List lastTariffs) { + this.lastTariffs = lastTariffs; + } } diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/constant/QuotaTypes.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/constant/QuotaTypes.java index 947183577a8..0da0d6e53f7 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/constant/QuotaTypes.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/constant/QuotaTypes.java @@ -20,9 +20,11 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; +import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.usage.UsageTypes; import org.apache.cloudstack.usage.UsageUnitTypes; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; +import org.apache.commons.lang3.StringUtils; public class QuotaTypes extends UsageTypes { private final Integer quotaType; @@ -106,6 +108,20 @@ public class QuotaTypes extends UsageTypes { return quotaTypeMap.get(quotaType); } + static public QuotaTypes getQuotaTypeByName(String name) { + if (StringUtils.isBlank(name)) { + throw new CloudRuntimeException("Could not retrieve Quota type by name because the value passed as parameter is null, empty, or blank."); + } + + for (QuotaTypes type : quotaTypeMap.values()) { + if (type.getQuotaName().equals(name)) { + return type; + } + } + + throw new CloudRuntimeException(String.format("Could not find Quota type with name [%s].", name)); + } + @Override public String toString() { return ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, "quotaType", "quotaName"); diff --git a/framework/rest/pom.xml b/framework/rest/pom.xml index d1ffff3c7bd..37a9791bd38 100644 --- a/framework/rest/pom.xml +++ b/framework/rest/pom.xml @@ -22,7 +22,7 @@ org.apache.cloudstack cloudstack-framework - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../pom.xml cloud-framework-rest diff --git a/framework/security/pom.xml b/framework/security/pom.xml index f41d5460bb7..05a108b4d45 100644 --- a/framework/security/pom.xml +++ b/framework/security/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-framework - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../pom.xml diff --git a/framework/spring/lifecycle/pom.xml b/framework/spring/lifecycle/pom.xml index fbdb2e60dc6..7c648d225cf 100644 --- a/framework/spring/lifecycle/pom.xml +++ b/framework/spring/lifecycle/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../../../pom.xml diff --git a/framework/spring/module/pom.xml b/framework/spring/module/pom.xml index ea39e3a6141..0a2280bff44 100644 --- a/framework/spring/module/pom.xml +++ b/framework/spring/module/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../../../pom.xml diff --git a/plugins/acl/dynamic-role-based/pom.xml b/plugins/acl/dynamic-role-based/pom.xml index b1972a54ba5..15150a26288 100644 --- a/plugins/acl/dynamic-role-based/pom.xml +++ b/plugins/acl/dynamic-role-based/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../../pom.xml diff --git a/plugins/acl/project-role-based/pom.xml b/plugins/acl/project-role-based/pom.xml index 3f5d64d29a7..4745936fc8b 100644 --- a/plugins/acl/project-role-based/pom.xml +++ b/plugins/acl/project-role-based/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../../pom.xml diff --git a/plugins/acl/static-role-based/pom.xml b/plugins/acl/static-role-based/pom.xml index 62fb60395e0..158add1395b 100644 --- a/plugins/acl/static-role-based/pom.xml +++ b/plugins/acl/static-role-based/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../../pom.xml diff --git a/plugins/affinity-group-processors/explicit-dedication/pom.xml b/plugins/affinity-group-processors/explicit-dedication/pom.xml index d6827ee13b6..d7535ba5925 100644 --- a/plugins/affinity-group-processors/explicit-dedication/pom.xml +++ b/plugins/affinity-group-processors/explicit-dedication/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../../pom.xml diff --git a/plugins/affinity-group-processors/host-affinity/pom.xml b/plugins/affinity-group-processors/host-affinity/pom.xml index bd999288717..68c66c41f39 100644 --- a/plugins/affinity-group-processors/host-affinity/pom.xml +++ b/plugins/affinity-group-processors/host-affinity/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../../pom.xml diff --git a/plugins/affinity-group-processors/host-anti-affinity/pom.xml b/plugins/affinity-group-processors/host-anti-affinity/pom.xml index b224bbaf34a..ba0dde5949d 100644 --- a/plugins/affinity-group-processors/host-anti-affinity/pom.xml +++ b/plugins/affinity-group-processors/host-anti-affinity/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../../pom.xml diff --git a/plugins/affinity-group-processors/non-strict-host-affinity/pom.xml b/plugins/affinity-group-processors/non-strict-host-affinity/pom.xml index bf751ca4141..352cf299dd7 100644 --- a/plugins/affinity-group-processors/non-strict-host-affinity/pom.xml +++ b/plugins/affinity-group-processors/non-strict-host-affinity/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../../pom.xml diff --git a/plugins/affinity-group-processors/non-strict-host-anti-affinity/pom.xml b/plugins/affinity-group-processors/non-strict-host-anti-affinity/pom.xml index 445acfc11d6..8896eba19e7 100644 --- a/plugins/affinity-group-processors/non-strict-host-anti-affinity/pom.xml +++ b/plugins/affinity-group-processors/non-strict-host-anti-affinity/pom.xml @@ -32,7 +32,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../../pom.xml diff --git a/plugins/alert-handlers/snmp-alerts/pom.xml b/plugins/alert-handlers/snmp-alerts/pom.xml index fad47d426f2..a64fc56ecbf 100644 --- a/plugins/alert-handlers/snmp-alerts/pom.xml +++ b/plugins/alert-handlers/snmp-alerts/pom.xml @@ -24,7 +24,7 @@ cloudstack-plugins org.apache.cloudstack - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../../pom.xml diff --git a/plugins/alert-handlers/syslog-alerts/pom.xml b/plugins/alert-handlers/syslog-alerts/pom.xml index 54641bd8a8a..957d99fd522 100644 --- a/plugins/alert-handlers/syslog-alerts/pom.xml +++ b/plugins/alert-handlers/syslog-alerts/pom.xml @@ -24,7 +24,7 @@ cloudstack-plugins org.apache.cloudstack - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../../pom.xml diff --git a/plugins/api/discovery/pom.xml b/plugins/api/discovery/pom.xml index 6426dcd70a5..e947834c3fa 100644 --- a/plugins/api/discovery/pom.xml +++ b/plugins/api/discovery/pom.xml @@ -25,7 +25,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../../pom.xml diff --git a/plugins/api/rate-limit/pom.xml b/plugins/api/rate-limit/pom.xml index 2449a23f2d0..2daba3ff488 100644 --- a/plugins/api/rate-limit/pom.xml +++ b/plugins/api/rate-limit/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../../pom.xml diff --git a/plugins/api/solidfire-intg-test/pom.xml b/plugins/api/solidfire-intg-test/pom.xml index 907c5f2968d..c2385a705c6 100644 --- a/plugins/api/solidfire-intg-test/pom.xml +++ b/plugins/api/solidfire-intg-test/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../../pom.xml diff --git a/plugins/api/vmware-sioc/pom.xml b/plugins/api/vmware-sioc/pom.xml index b3c04e603cb..858e7ea69a0 100644 --- a/plugins/api/vmware-sioc/pom.xml +++ b/plugins/api/vmware-sioc/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../../pom.xml diff --git a/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/BackrollBackupProvider.java b/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/BackrollBackupProvider.java index 37a588fdc90..a74bfc5e931 100644 --- a/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/BackrollBackupProvider.java +++ b/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/BackrollBackupProvider.java @@ -35,6 +35,8 @@ import org.apache.cloudstack.backup.backroll.BackrollService; import org.apache.cloudstack.backup.backroll.model.BackrollBackupMetrics; import org.apache.cloudstack.backup.backroll.model.BackrollTaskStatus; import org.apache.cloudstack.backup.backroll.model.BackrollVmBackup; +import org.apache.cloudstack.backup.backroll.utils.BackrollApiException; +import org.apache.cloudstack.backup.backroll.utils.BackrollHttpClientProvider; import org.apache.cloudstack.backup.dao.BackupDao; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; @@ -90,7 +92,6 @@ public class BackrollBackupProvider extends AdapterBase implements BackupProvide @Override public List listBackupOfferings(Long zoneId) { logger.debug("Listing backup policies on backroll B&R Plugin"); - logger.info("Listing backup policies on backroll B&R Plugin"); BackrollClient client = getClient(zoneId); try{ String urlToRequest = client.getBackupOfferingUrl(); @@ -106,8 +107,8 @@ public class BackrollBackupProvider extends AdapterBase implements BackupProvide } return results; } - } catch (KeyManagementException | ParseException | NoSuchAlgorithmException | IOException e) { - logger.info("BackrollProvider: catch erreur!!!!!!!!!!!!!!!!!!!!!!!!"); + } catch (ParseException | BackrollApiException | IOException e) { + logger.info("BackrollProvider: catch erreur: " + e); throw new CloudRuntimeException("Failed to load backup offerings"); } return new ArrayList(); @@ -122,8 +123,11 @@ public class BackrollBackupProvider extends AdapterBase implements BackupProvide @Override public boolean assignVMToBackupOffering(VirtualMachine vm, BackupOffering backupOffering) { logger.info("Creating VM backup for VM {} from backup offering {}", vm.getInstanceName(), backupOffering.getName()); - ((VMInstanceVO) vm).setBackupExternalId(backupOffering.getUuid()); - return true; + if(vm instanceof VMInstanceVO) { + ((VMInstanceVO) vm).setBackupExternalId(backupOffering.getUuid()); + return true; + } + return false; } @Override @@ -132,7 +136,7 @@ public class BackrollBackupProvider extends AdapterBase implements BackupProvide boolean isSuccess; try { isSuccess = getClient(vm.getDataCenterId()).restoreVMFromBackup(vm.getUuid(), getBackupName(backup)); - } catch (KeyManagementException | ParseException | NoSuchAlgorithmException | IOException e) { + } catch (ParseException | BackrollApiException | IOException e) { throw new CloudRuntimeException("Failed to restore VM from Backrup"); } return isSuccess; @@ -149,6 +153,7 @@ public class BackrollBackupProvider extends AdapterBase implements BackupProvide List vmUuids = vms.stream().filter(Objects::nonNull).map(VirtualMachine::getUuid).collect(Collectors.toList()); logger.debug("Get Backup Metrics for VMs: {}.", String.join(", ", vmUuids)); + BackrollClient client = getClient(zoneId); for (final VirtualMachine vm : vms) { if (vm == null) { continue; @@ -156,8 +161,8 @@ public class BackrollBackupProvider extends AdapterBase implements BackupProvide Metric metric; try { - metric = getClient(zoneId).getVirtualMachineMetrics(vm.getUuid()); - } catch (KeyManagementException | NoSuchAlgorithmException | IOException e) { + metric = client.getVirtualMachineMetrics(vm.getUuid()); + } catch (BackrollApiException | IOException e) { throw new CloudRuntimeException("Failed to retrieve backup metrics"); } logger.debug("Metrics for VM [uuid: {}, name: {}] is [backup size: {}, data size: {}].", vm.getUuid(), @@ -216,14 +221,15 @@ public class BackrollBackupProvider extends AdapterBase implements BackupProvide logger.info("Starting sync backup for VM ID " + vm.getUuid() + " on backroll provider"); final BackrollClient client = getClient(vm.getDataCenterId()); - List backupsInDb = backupDao.listByVmId(null, vm.getId()); + List backupsInDb = backupDao.listByVmId(vm.getDataCenterId(), vm.getId()); for (Backup backup : backupsInDb) { if (backup.getStatus().equals(Backup.Status.BackingUp)) { BackrollTaskStatus response; try { response = client.checkBackupTaskStatus(backup.getExternalId()); - } catch (KeyManagementException | ParseException | NoSuchAlgorithmException | IOException e) { + } catch (ParseException | BackrollApiException | IOException e) { + logger.error(e); throw new CloudRuntimeException("Failed to sync backups"); } @@ -231,17 +237,7 @@ public class BackrollBackupProvider extends AdapterBase implements BackupProvide logger.debug("backroll backup id: {}", backup.getExternalId()); logger.debug("backroll backup status: {}", response.getState()); - BackupVO backupToUpdate = new BackupVO(); - backupToUpdate.setVmId(backup.getVmId()); - backupToUpdate.setExternalId(backup.getExternalId()); - backupToUpdate.setType(backup.getType()); - backupToUpdate.setDate(backup.getDate()); - backupToUpdate.setSize(backup.getSize()); - backupToUpdate.setProtectedSize(backup.getProtectedSize()); - backupToUpdate.setBackupOfferingId(vm.getBackupOfferingId()); - backupToUpdate.setAccountId(backup.getAccountId()); - backupToUpdate.setDomainId(backup.getDomainId()); - backupToUpdate.setZoneId(backup.getZoneId()); + BackupVO backupToUpdate = ((BackupVO) backup); if (response.getState().equals("PENDING")) { backupToUpdate.setStatus(Backup.Status.BackingUp); @@ -259,6 +255,8 @@ public class BackrollBackupProvider extends AdapterBase implements BackupProvide backupToUpdate.setProtectedSize(backupMetrics.getSize()); // total size } } catch (KeyManagementException | NoSuchAlgorithmException | IOException e) { + } catch (BackrollApiException | IOException e) { + logger.error(e); throw new CloudRuntimeException("Failed to get backup metrics"); } } else { @@ -267,7 +265,6 @@ public class BackrollBackupProvider extends AdapterBase implements BackupProvide if (backupDao.persist(backupToUpdate) != null) { logger.info("Backroll mise à jour enregistrée"); - backupDao.remove(backup.getId()); } } } else if (backup.getStatus().equals(Backup.Status.BackedUp) && backup.getSize().equals(0L)) { @@ -275,7 +272,8 @@ public class BackrollBackupProvider extends AdapterBase implements BackupProvide BackrollBackupMetrics backupMetrics; try { backupMetrics = client.getBackupMetrics(vm.getUuid() , backupId); - } catch (KeyManagementException | NoSuchAlgorithmException | IOException e) { + } catch (BackrollApiException | IOException e) { + logger.error(e); throw new CloudRuntimeException("Failed to get backup metrics"); } @@ -289,57 +287,63 @@ public class BackrollBackupProvider extends AdapterBase implements BackupProvide } // Backups synchronisation between Backroll ad CS Db - List backupsFromBackroll = client.getAllBackupsfromVirtualMachine(vm.getUuid()); - backupsInDb = backupDao.listByVmId(null, vm.getId()); + List backupsFromBackroll; + try { + backupsFromBackroll = client.getAllBackupsfromVirtualMachine(vm.getUuid()); - // insert new backroll backup in CS - for (BackrollVmBackup backupInBackroll : backupsFromBackroll) { - Backup backupToFind = backupsInDb.stream() - .filter(backupInDb -> backupInDb.getExternalId().contains(backupInBackroll.getName())) - .findAny() - .orElse(null); + backupsInDb = backupDao.listByVmId(null, vm.getId()); - if (backupToFind == null) { - BackupVO backupToInsert = new BackupVO(); - backupToInsert.setVmId(vm.getId()); - backupToInsert.setExternalId(backupInBackroll.getId() + "," + backupInBackroll.getName()); - backupToInsert.setType("INCREMENTAL"); - backupToInsert.setDate(backupInBackroll.getDate()); - backupToInsert.setSize(0L); - backupToInsert.setProtectedSize(0L); - backupToInsert.setStatus(Backup.Status.BackedUp); - backupToInsert.setBackupOfferingId(vm.getBackupOfferingId()); - backupToInsert.setAccountId(vm.getAccountId()); - backupToInsert.setDomainId(vm.getDomainId()); - backupToInsert.setZoneId(vm.getDataCenterId()); - backupDao.persist(backupToInsert); - } - if (backupToFind != null && backupToFind.getStatus() == Backup.Status.Removed) { - BackupVO backupToUpdate = ((BackupVO) backupToFind); - backupToUpdate.setStatus(Backup.Status.BackedUp); - if (backupDao.persist(backupToUpdate) != null) { - logger.info("Backroll update saved"); - backupDao.remove(backupToFind.getId()); + // insert new backroll backup in CS + for (BackrollVmBackup backupInBackroll : backupsFromBackroll) { + Backup backupToFind = backupsInDb.stream() + .filter(backupInDb -> backupInDb.getExternalId().contains(backupInBackroll.getName())) + .findAny() + .orElse(null); + + if (backupToFind == null) { + BackupVO backupToInsert = new BackupVO(); + backupToInsert.setVmId(vm.getId()); + backupToInsert.setExternalId(backupInBackroll.getId() + "," + backupInBackroll.getName()); + backupToInsert.setType("INCREMENTAL"); + backupToInsert.setDate(backupInBackroll.getDate()); + backupToInsert.setSize(0L); + backupToInsert.setProtectedSize(0L); + backupToInsert.setStatus(Backup.Status.BackedUp); + backupToInsert.setBackupOfferingId(vm.getBackupOfferingId()); + backupToInsert.setAccountId(vm.getAccountId()); + backupToInsert.setDomainId(vm.getDomainId()); + backupToInsert.setZoneId(vm.getDataCenterId()); + backupDao.persist(backupToInsert); + } + if (backupToFind != null && backupToFind.getStatus() == Backup.Status.Removed) { + BackupVO backupToUpdate = ((BackupVO) backupToFind); + backupToUpdate.setStatus(Backup.Status.BackedUp); + if (backupDao.persist(backupToUpdate) != null) { + logger.info("Backroll update saved"); + backupDao.remove(backupToFind.getId()); + } } } - } - // delete deleted backroll backup in CS - backupsInDb = backupDao.listByVmId(null, vm.getId()); - for (Backup backup : backupsInDb) { - String backupName = backup.getExternalId().contains(",") ? backup.getExternalId().split(",")[1] : backup.getExternalId(); - BackrollVmBackup backupToFind = backupsFromBackroll.stream() - .filter(backupInBackroll -> backupInBackroll.getName().contains(backupName)) - .findAny() - .orElse(null); + // delete deleted backroll backup in CS + backupsInDb = backupDao.listByVmId(null, vm.getId()); + for (Backup backup : backupsInDb) { + String backupName = backup.getExternalId().contains(",") ? backup.getExternalId().split(",")[1] : backup.getExternalId(); + BackrollVmBackup backupToFind = backupsFromBackroll.stream() + .filter(backupInBackroll -> backupInBackroll.getName().contains(backupName)) + .findAny() + .orElse(null); - if (backupToFind == null) { - BackupVO backupToUpdate = ((BackupVO) backup); - backupToUpdate.setStatus(Backup.Status.Removed); - if (backupDao.persist(backupToUpdate) != null) { - logger.debug("Backroll delete saved (sync)"); + if (backupToFind == null) { + BackupVO backupToUpdate = ((BackupVO) backup); + backupToUpdate.setStatus(Backup.Status.Removed); + if (backupDao.persist(backupToUpdate) != null) { + logger.debug("Backroll delete saved (sync)"); + } } } + } catch (BackrollApiException | IOException e) { + logger.error(e); } } @@ -374,7 +378,8 @@ public class BackrollBackupProvider extends AdapterBase implements BackupProvide logger.debug("Backup deletion for backup {} complete on backroll side.", backup.getUuid()); return deleteBackupInDb(backup); } - } catch (KeyManagementException | NoSuchAlgorithmException | IOException e) { + } catch (BackrollApiException | IOException e) { + logger.error(e); throw new CloudRuntimeException("Failed to delete backup"); } } @@ -397,15 +402,17 @@ public class BackrollBackupProvider extends AdapterBase implements BackupProvide try { if (backrollClient == null) { logger.debug("backroll client null - instanciation of new one "); - backrollClient = new BackrollClient(BackrollUrlConfigKey.valueIn(zoneId), BackrollAppNameConfigKey.valueIn(zoneId), BackrollPasswordConfigKey.valueIn(zoneId), true, 300, 600, new BackrollService()); + BackrollHttpClientProvider provider = new BackrollHttpClientProvider(BackrollUrlConfigKey.valueIn(zoneId), BackrollAppNameConfigKey.valueIn(zoneId), BackrollPasswordConfigKey.valueIn(zoneId), true, 300, 600); + backrollClient = new BackrollClient(provider); } return backrollClient; } catch (URISyntaxException e) { + logger.error(e); throw new CloudRuntimeException("Failed to parse Backroll API URL: " + e.getMessage()); } catch (NoSuchAlgorithmException | KeyManagementException e) { - logger.info("Failed to build Backroll API client due to: ", e); + logger.error(e); + throw new CloudRuntimeException("Failed to build Backroll API client"); } - throw new CloudRuntimeException("Failed to build Backroll API client"); } private String getBackupName(Backup backup) { diff --git a/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/BackrollClient.java b/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/BackrollClient.java index 891927736d0..21e7fca2ac7 100644 --- a/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/BackrollClient.java +++ b/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/BackrollClient.java @@ -17,20 +17,11 @@ package org.apache.cloudstack.backup.backroll; import java.io.IOException; -import java.net.URI; import java.net.URISyntaxException; import java.security.KeyManagementException; import java.security.NoSuchAlgorithmException; -import java.security.SecureRandom; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.TimeUnit; - -import javax.net.ssl.SSLContext; -import javax.net.ssl.X509TrustManager; - -import org.apache.cloudstack.api.ApiErrorCode; -import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.backup.BackupOffering; import org.apache.cloudstack.backup.Backup.Metric; import org.apache.cloudstack.backup.backroll.BackrollService.NotOkBodyException; @@ -41,8 +32,6 @@ import org.apache.cloudstack.backup.backroll.model.BackrollTaskStatus; import org.apache.cloudstack.backup.backroll.model.BackrollVmBackup; import org.apache.cloudstack.backup.backroll.model.response.BackrollTaskRequestResponse; import org.apache.cloudstack.backup.backroll.model.response.TaskState; -import org.apache.cloudstack.backup.backroll.model.response.api.LoginApiResponse; -import org.apache.cloudstack.backup.backroll.model.response.archive.BackrollArchiveResponse; import org.apache.cloudstack.backup.backroll.model.response.archive.BackrollBackupsFromVMResponse; import org.apache.cloudstack.backup.backroll.model.response.backup.BackrollBackupStatusResponse; import org.apache.cloudstack.backup.backroll.model.response.backup.BackrollBackupStatusSuccessResponse; @@ -53,28 +42,11 @@ import org.apache.cloudstack.backup.backroll.model.response.metrics.virtualMachi import org.apache.cloudstack.backup.backroll.model.response.metrics.virtualMachineBackups.VirtualMachineBackupsResponse; import org.apache.cloudstack.backup.backroll.model.response.policy.BackrollBackupPolicyResponse; import org.apache.cloudstack.backup.backroll.model.response.policy.BackupPoliciesResponse; -import org.apache.cloudstack.utils.security.SSLUtils; +import org.apache.cloudstack.backup.backroll.utils.BackrollApiException; +import org.apache.cloudstack.backup.backroll.utils.BackrollHttpClientProvider; import org.apache.commons.lang3.StringUtils; -import org.apache.http.HttpEntity; -import org.apache.http.HttpHeaders; -import org.apache.http.HttpResponse; -import org.apache.http.HttpStatus; -import org.apache.http.ParseException; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpDelete; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.conn.ssl.NoopHostnameVerifier; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; -import org.apache.http.entity.ContentType; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClientBuilder; -import org.apache.http.util.EntityUtils; - import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -82,12 +54,6 @@ import org.joda.time.DateTime; import org.json.JSONException; import org.json.JSONObject; - -import com.cloud.utils.exception.CloudRuntimeException; -import com.cloud.utils.nio.TrustAllManager; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; @@ -96,425 +62,171 @@ public class BackrollClient { protected Logger logger = LogManager.getLogger(BackrollClient.class); - private final URI apiURI; + private BackrollHttpClientProvider httpProvider; - private String backrollToken = null; - private String appname = null; - private String password = null; - private boolean validateCertificate = false; - private RequestConfig config = null; - private BackrollService backrollService; - - private int restoreTimeout; - - public BackrollClient(final String url, final String appname, final String password, - final boolean validateCertificate, final int timeout, - final int restoreTimeout, final BackrollService backrollService) + public BackrollClient(BackrollHttpClientProvider httpProvider) throws URISyntaxException, NoSuchAlgorithmException, KeyManagementException { - this.apiURI = new URI(url); - this.restoreTimeout = restoreTimeout; - this.appname = appname; - this.password = password; - this.backrollService = backrollService; - this.config = RequestConfig.custom() - .setConnectTimeout(timeout * 1000) - .setConnectionRequestTimeout(timeout * 1000) - .setSocketTimeout(timeout * 1000) - .build(); + this.httpProvider = httpProvider; } - private CloseableHttpClient createHttpClient() throws KeyManagementException, NoSuchAlgorithmException { - if(!validateCertificate) { - SSLContext sslContext = SSLUtils.getSSLContext(); - sslContext.init(null, new X509TrustManager[] { new TrustAllManager() }, new SecureRandom()); - final SSLConnectionSocketFactory factory = new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE); - return HttpClientBuilder.create() - .setDefaultRequestConfig(config) - .setSSLSocketFactory(factory) - .build(); - } else { - return HttpClientBuilder.create() - .setDefaultRequestConfig(config) - .build(); - } + public String startBackupJob(final String jobId) throws IOException, BackrollApiException { + logger.info("Trying to start backup for Backroll job: {}", jobId); + String backupJob = ""; + BackrollTaskRequestResponse requestResponse = httpProvider.post(String.format("/tasks/singlebackup/%s", jobId), null, BackrollTaskRequestResponse.class); + backupJob = requestResponse.location.replace("/api/v1/status/", ""); + return StringUtils.isEmpty(backupJob) ? null : backupJob; } - private void closeConnection(CloseableHttpResponse closeableHttpResponse) throws IOException { - closeableHttpResponse.close(); - } - - private String triggerBackrollTaskForVM(final String vmId, JSONObject jsonBody, final String task) throws InterruptedException, KeyManagementException, ParseException, NoSuchAlgorithmException, IOException, NotOkBodyException { - CloseableHttpResponse response = backrollService.post(apiURI,String.format("/tasks/%s/%s",task, vmId) ,jsonBody); - String result = backrollService.okBody(response); - BackrollTaskRequestResponse requestResponse = new ObjectMapper().readValue(result, BackrollTaskRequestResponse.class); - response.close(); - String urlToRequest = requestResponse.location.replace("/api/v1", ""); - return urlToRequest; - } - - public boolean restoreVMFromBackup(final String vmId, final String backupName) throws KeyManagementException, ParseException, NoSuchAlgorithmException, IOException { - logger.info("Start restore backup with backroll with backup {} for vm {}", backupName, vmId); - - loginIfAuthenticationFailed(); - boolean isRestoreOk = false; - - try { - JSONObject jsonBody = new JSONObject(); - try { - jsonBody.put("virtual_machine_id", vmId); - jsonBody.put("backup_name", backupName); - jsonBody.put("storage", ""); - jsonBody.put("mode", "single"); - - } catch (JSONException e) { - logger.error("Backroll Error: {}", e.getMessage()); - } - - String urlToRequest = triggerBackrollTaskForVM(vmId, jsonBody, "restore"); - String result = backrollService.waitGet(apiURI,urlToRequest); - - if(result.contains("SUCCESS")) { - logger.debug("RESTORE SUCCESS"); - isRestoreOk = true; - } - - } catch (final NotOkBodyException e) { - return false; - } catch (final IOException | InterruptedException e) { - logger.error("Ouch! Failed to restore VM with Backroll due to: {}", e.getMessage()); - throw new CloudRuntimeException("Ouch! Failed to restore VM with Backroll due to: {}" + e.getMessage()); - } - return isRestoreOk; - } - - public void triggerTaskStatus(String urlToRequest) - throws IOException, InterruptedException, KeyManagementException, ParseException, NoSuchAlgorithmException{ - backrollService.waitGet(apiURI,urlToRequest); - } - - public String startBackupJob(final String vmId) throws KeyManagementException, NoSuchAlgorithmException, IOException { - logger.info("Trying to start backup for Backroll job: {}", vmId); - loginIfAuthenticationFailed(); - - try { - String urlToRequest = triggerBackrollTaskForVM(vmId, null, "singlebackup"); - return urlToRequest; - } catch (final Exception e) { - logger.error("Failed to start Backroll backup job due to: {}", e.getMessage()); - } - return null; - } - - public String getBackupOfferingUrl() throws KeyManagementException, NoSuchAlgorithmException, IOException { - logger.info("Trying to list backroll backup policies"); - - loginIfAuthenticationFailed(); + public String getBackupOfferingUrl() throws IOException, BackrollApiException { + logger.info("Trying to get backroll backup policies url"); String url = ""; - - try { - CloseableHttpResponse response = backrollService.get(apiURI,"/backup_policies"); - String result = backrollService.okBody(response); - logger.info("BackrollClient:getBackupOfferingUrl:result: " + result); - //BackrollTaskRequestResponse requestResponse = parse(result); - BackrollTaskRequestResponse requestResponse = new ObjectMapper().readValue(result, BackrollTaskRequestResponse.class); - logger.info("BackrollClient:getBackupOfferingUrl:Apres PArse: " + requestResponse.location); - response.close(); - url = requestResponse.location.replace("/api/v1", ""); - } catch (final Exception e) { - logger.info("Failed to list Backroll jobs due to: {}", e.getMessage()); - logger.error("Failed to list Backroll jobs due to: {}", e.getMessage()); - } + BackrollTaskRequestResponse requestResponse = httpProvider.get("/backup_policies", BackrollTaskRequestResponse.class); + logger.info("BackrollClient:getBackupOfferingUrl:Apres Parse: " + requestResponse.location); + url = requestResponse.location.replace("/api/v1", ""); return StringUtils.isEmpty(url) ? null : url; } - public List getBackupOfferings(String idTask) throws KeyManagementException, ParseException, NoSuchAlgorithmException, IOException { + public List getBackupOfferings(String idTask) throws BackrollApiException, IOException { logger.info("Trying to list backroll backup policies"); - - loginIfAuthenticationFailed(); final List policies = new ArrayList<>(); - - try { - String results = backrollService.waitGet(apiURI,idTask); - BackupPoliciesResponse backupPoliciesResponse = new ObjectMapper().readValue(results, BackupPoliciesResponse.class); - - for (final BackrollBackupPolicyResponse policy : backupPoliciesResponse.backupPolicies) { - policies.add(new BackrollOffering(policy.name, policy.id)); - } - } catch (final IOException | InterruptedException e) { - logger.error("Failed to list Backroll jobs due to: {}", e.getMessage()); + BackupPoliciesResponse backupPoliciesResponse = httpProvider.waitGet(idTask, BackupPoliciesResponse.class); + logger.info("BackrollClient:getBackupOfferings:Apres Parse: " + backupPoliciesResponse.backupPolicies.get(0).name); + for (final BackrollBackupPolicyResponse policy : backupPoliciesResponse.backupPolicies) { + policies.add(new BackrollOffering(policy.name, policy.id)); } + return policies; } - public BackrollTaskStatus checkBackupTaskStatus(String taskId) throws KeyManagementException, ParseException, NoSuchAlgorithmException, IOException { - logger.info("Trying to get backup status for Backroll task: {}", taskId); + public boolean restoreVMFromBackup(final String vmId, final String backupName) throws IOException, BackrollApiException { + logger.info("Start restore backup with backroll with backup {} for vm {}", backupName, vmId); - loginIfAuthenticationFailed(); + boolean isRestoreOk = false; + + JSONObject jsonBody = new JSONObject(); + try { + jsonBody.put("virtual_machine_id", vmId); + jsonBody.put("backup_name", backupName); + jsonBody.put("storage", ""); + jsonBody.put("mode", "single"); + + } catch (JSONException e) { + logger.error("Backroll Error: {}", e.getMessage()); + } + + BackrollTaskRequestResponse requestResponse = httpProvider.post(String.format("/tasks/restore/%s", vmId), jsonBody, BackrollTaskRequestResponse.class); + String urlToRequest = requestResponse.location.replace("/api/v1", ""); + + String result = httpProvider.waitGetWithoutParseResponse(urlToRequest); + if(result.contains("SUCCESS")) { + logger.debug("RESTORE SUCCESS content : " + result); + logger.debug("RESTORE SUCCESS"); + isRestoreOk = true; + } + + return isRestoreOk; + } + + public BackrollTaskStatus checkBackupTaskStatus(String taskId) throws IOException, BackrollApiException { + logger.info("Trying to get backup status for Backroll task: {}", taskId); BackrollTaskStatus status = new BackrollTaskStatus(); - try { + String backupResponse = httpProvider.getWithoutParseResponse("/status/" + taskId); - String body = backrollService.okBody(backrollService.get(apiURI,"/status/" + taskId)); - - if (body.contains(TaskState.FAILURE) || body.contains(TaskState.PENDING)) { - BackrollBackupStatusResponse backupStatusRequestResponse = new ObjectMapper().readValue(body, BackrollBackupStatusResponse.class); - status.setState(backupStatusRequestResponse.state); - } else { - BackrollBackupStatusSuccessResponse backupStatusSuccessRequestResponse = new ObjectMapper().readValue(body, BackrollBackupStatusSuccessResponse.class); - status.setState(backupStatusSuccessRequestResponse.state); - status.setInfo(backupStatusSuccessRequestResponse.info); - } - - } catch (final NotOkBodyException e) { - // throw new CloudRuntimeException("Failed to retrieve backups status for this - // VM via Backroll"); - logger.error("Failed to retrieve backups status for this VM via Backroll"); - - } catch (final IOException e) { - logger.error("Failed to check backups status due to: {}", e.getMessage()); + if (backupResponse.contains(TaskState.FAILURE) || backupResponse.contains(TaskState.PENDING)) { + BackrollBackupStatusResponse backupStatusRequestResponse = new ObjectMapper().readValue(backupResponse, BackrollBackupStatusResponse.class); + status.setState(backupStatusRequestResponse.state); + } else { + BackrollBackupStatusSuccessResponse backupStatusSuccessRequestResponse = new ObjectMapper().readValue(backupResponse, BackrollBackupStatusSuccessResponse.class); + status.setState(backupStatusSuccessRequestResponse.state); + status.setInfo(backupStatusSuccessRequestResponse.info); } + return StringUtils.isEmpty(status.getState()) ? null : status; } - public boolean deleteBackup(final String vmId, final String backupName) throws KeyManagementException, NoSuchAlgorithmException, IOException { + public boolean deleteBackup(final String vmId, final String backupName) throws IOException, BackrollApiException{ logger.info("Trying to delete backup {} for vm {} using Backroll", vmId, backupName); - - loginIfAuthenticationFailed(); - boolean isBackupDeleted = false; - try { + BackrollTaskRequestResponse requestResponse = httpProvider.delete(String.format("/virtualmachines/%s/backups/%s", vmId, backupName), BackrollTaskRequestResponse.class); + String urlToRequest = requestResponse.location.replace("/api/v1", ""); - CloseableHttpResponse response = backrollService.delete(apiURI,String.format("/virtualmachines/%s/backups/%s", vmId, backupName)); - String result = backrollService.okBody(response); - BackrollTaskRequestResponse requestResponse = new ObjectMapper().readValue(result, BackrollTaskRequestResponse.class); - response.close(); - String urlToRequest = requestResponse.location.replace("/api/v1", ""); + BackrollBackupsFromVMResponse backrollBackupsFromVMResponse = httpProvider.waitGet(urlToRequest, BackrollBackupsFromVMResponse.class); + logger.debug(backrollBackupsFromVMResponse.state); + isBackupDeleted = backrollBackupsFromVMResponse.state.equals(TaskState.SUCCESS); - result = backrollService.waitGet(apiURI,urlToRequest); - BackrollBackupsFromVMResponse backrollBackupsFromVMResponse = new ObjectMapper().readValue(result, BackrollBackupsFromVMResponse.class); - logger.debug(backrollBackupsFromVMResponse.state); - isBackupDeleted = backrollBackupsFromVMResponse.state.equals(TaskState.SUCCESS); - } catch (final NotOkBodyException e) { - isBackupDeleted = false; - } catch (final Exception e) { - isBackupDeleted = false; - logger.error("Failed to delete backup using Backroll due to: {}", e.getMessage()); - } return isBackupDeleted; } - public Metric getVirtualMachineMetrics(final String vmId) throws KeyManagementException, NoSuchAlgorithmException, IOException { + public Metric getVirtualMachineMetrics(final String vmId) throws IOException, BackrollApiException { logger.info("Trying to retrieve virtual machine metric from Backroll for vm {}", vmId); - loginIfAuthenticationFailed(); - Metric metric = new Metric(0L, 0L); - try { + BackrollTaskRequestResponse requestResponse = httpProvider.get(String.format("/virtualmachines/%s/repository", vmId), BackrollTaskRequestResponse.class); - CloseableHttpResponse response = backrollService.get(apiURI,String.format("/virtualmachines/%s/repository", vmId)); - String result = backrollService.okBody(response); - BackrollTaskRequestResponse requestResponse = new ObjectMapper().readValue(result, BackrollTaskRequestResponse.class); - response.close(); - String urlToRequest = requestResponse.location.replace("/api/v1", ""); + String urlToRequest = requestResponse.location.replace("/api/v1", ""); - result = backrollService.waitGet(apiURI,urlToRequest); - BackrollVmMetricsResponse vmMetricsResponse =new ObjectMapper().readValue(result, BackrollVmMetricsResponse.class); + BackrollVmMetricsResponse vmMetricsResponse = httpProvider.waitGet(urlToRequest, BackrollVmMetricsResponse.class); - if (vmMetricsResponse != null && vmMetricsResponse.state.equals(TaskState.SUCCESS)) { - logger.debug("SUCCESS ok"); - CacheStats stats = null; - try { - stats = vmMetricsResponse.infos.cache.stats; - } catch (NullPointerException e) { - } - if (stats != null) { - long size = Long.parseLong(stats.totalSize); - metric = new Metric(size, size); - } + if (vmMetricsResponse != null && vmMetricsResponse.state.equals(TaskState.SUCCESS)) { + logger.debug("SUCCESS ok"); + CacheStats stats = null; + try { + stats = vmMetricsResponse.infos.cache.stats; + } catch (NullPointerException e) { + } + if (stats != null) { + long size = Long.parseLong(stats.totalSize); + metric = new Metric(size, size); } - } catch (final Exception e) { - logger.error("Failed to retrieve virtual machine metrics with Backroll due to: {}", e.getMessage()); } return metric; } - public BackrollBackupMetrics getBackupMetrics(String vmId, String backupId) throws KeyManagementException, NoSuchAlgorithmException, IOException { + public BackrollBackupMetrics getBackupMetrics(String vmId, String backupId) throws IOException, BackrollApiException { logger.info("Trying to get backup metrics for VM: {}, and backup: {}", vmId, backupId); - loginIfAuthenticationFailed(); - BackrollBackupMetrics metrics = null; - try { + BackrollTaskRequestResponse requestResponse = httpProvider.get(String.format("/virtualmachines/%s/backups/%s", vmId, backupId), BackrollTaskRequestResponse.class); - CloseableHttpResponse response = backrollService.get(apiURI,String.format("/virtualmachines/%s/backups/%s", vmId, backupId)); - String result = backrollService.okBody(response); - BackrollTaskRequestResponse requestResponse = new ObjectMapper().readValue(result, BackrollTaskRequestResponse.class); - response.close(); - String urlToRequest = requestResponse.location.replace("/api/v1", ""); + String urlToRequest = requestResponse.location.replace("/api/v1", ""); - logger.debug(urlToRequest); + logger.debug(urlToRequest); - result = backrollService.waitGet(apiURI,urlToRequest); - BackrollBackupMetricsResponse metricsResponse = new ObjectMapper().readValue(result, BackrollBackupMetricsResponse.class); - if (metricsResponse.info != null) { - metrics = new BackrollBackupMetrics(Long.parseLong(metricsResponse.info.originalSize), - Long.parseLong(metricsResponse.info.deduplicatedSize)); - } - } catch (final NotOkBodyException e) { - throw new CloudRuntimeException("Failed to retrieve backups status for this VM via Backroll"); - } catch (final IOException | InterruptedException e) { - logger.error("Failed to check backups status due to: {}", e.getMessage()); + BackrollBackupMetricsResponse metricsResponse = httpProvider.waitGet(urlToRequest, BackrollBackupMetricsResponse.class); + if (metricsResponse.info != null) { + metrics = new BackrollBackupMetrics(Long.parseLong(metricsResponse.info.originalSize), + Long.parseLong(metricsResponse.info.deduplicatedSize)); } return metrics; } - public List getAllBackupsfromVirtualMachine(String vmId) { - //logger.info("Trying to retrieve all backups for vm {}", vmId); + public List getAllBackupsfromVirtualMachine(String vmId) throws BackrollApiException, IOException { + logger.info("Trying to retrieve all backups for vm {}", vmId); List backups = new ArrayList(); - try { + BackrollTaskRequestResponse requestResponse = httpProvider.get(String.format("/virtualmachines/%s/backups", vmId), BackrollTaskRequestResponse.class); - CloseableHttpResponse response = backrollService.get(apiURI, String.format("/virtualmachines/%s/backups", vmId)); - String result = backrollService.okBody(response); - BackrollTaskRequestResponse requestResponse = new ObjectMapper().readValue(result, BackrollTaskRequestResponse.class); - response.close(); - String urlToRequest = requestResponse.location.replace("/api/v1", ""); + String urlToRequest = requestResponse.location.replace("/api/v1", ""); + logger.debug(urlToRequest); + VirtualMachineBackupsResponse virtualMachineBackupsResponse = httpProvider.waitGet(urlToRequest, VirtualMachineBackupsResponse.class); - //logger.debug(urlToRequest); - - result = backrollService.waitGet(apiURI,urlToRequest); - VirtualMachineBackupsResponse virtualMachineBackupsResponse = new ObjectMapper().readValue(result, VirtualMachineBackupsResponse.class); - - if (virtualMachineBackupsResponse.state.equals(TaskState.SUCCESS)) { - if (virtualMachineBackupsResponse.info.archives.size() > 0) { - for (BackupInfos infos : virtualMachineBackupsResponse.info.archives) { - var dateStart = new DateTime(infos.start); - backups.add(new BackrollVmBackup(infos.id, infos.name, dateStart.toDate())); - } + if (virtualMachineBackupsResponse.state.equals(TaskState.SUCCESS)) { + if (virtualMachineBackupsResponse.info.archives.size() > 0) { + for (BackupInfos infos : virtualMachineBackupsResponse.info.archives) { + var dateStart = new DateTime(infos.start); + backups.add(new BackrollVmBackup(infos.id, infos.name, dateStart.toDate())); } } - - } catch (Exception e) { - e.printStackTrace(); } + return backups; } - - private boolean isResponseAuthorized(final HttpResponse response) { - return response.getStatusLine().getStatusCode() == HttpStatus.SC_OK - || response.getStatusLine().getStatusCode() == HttpStatus.SC_ACCEPTED; - } - - private boolean isAuthenticated() throws KeyManagementException, NoSuchAlgorithmException { - boolean result = false; - - if(StringUtils.isEmpty(backrollToken)) { - return result; - } - - try { - CloseableHttpResponse response = backrollService.post(apiURI,"/auth", null); - result = isResponseAuthorized(response); - EntityUtils.consumeQuietly(response.getEntity()); - closeConnection(response); - } catch (IOException e) { - logger.error("Failed to authenticate to Backroll due to: {}", e.getMessage()); - } - return result; - } - - private void loginIfAuthenticationFailed() throws KeyManagementException, NoSuchAlgorithmException, IOException { - if (!isAuthenticated()) { - login(appname, password); - } - } - - - private void login(final String appname, final String appsecret) throws IOException, KeyManagementException, NoSuchAlgorithmException { - logger.debug("Backroll client - start login"); - - CloseableHttpClient httpClient = createHttpClient(); - CloseableHttpResponse response = null; - - final HttpPost request = new HttpPost(apiURI.toString() + "/login"); - request.addHeader("content-type", "application/json"); - - JSONObject jsonBody = new JSONObject(); - StringEntity params; - - try { - jsonBody.put("app_id", appname); - jsonBody.put("app_secret", appsecret); - params = new StringEntity(jsonBody.toString()); - request.setEntity(params); - - response = httpClient.execute(request); - try { - String toto = backrollService.okBody(response); - ObjectMapper objectMapper = new ObjectMapper(); - logger.info("BACKROLL: " + toto); - LoginApiResponse loginResponse = objectMapper.readValue(toto, LoginApiResponse.class); - logger.info("ok"); - backrollToken = loginResponse.accessToken; - logger.debug("Backroll client - Token : {}", backrollToken); - - if (StringUtils.isEmpty(loginResponse.accessToken)) { - throw new CloudRuntimeException("Backroll token is not available to perform API requests"); - } - } catch (final NotOkBodyException e) { - if (response.getStatusLine().getStatusCode() != HttpStatus.SC_CREATED) { - throw new CloudRuntimeException( - "Failed to create and authenticate Backroll client, please check the settings."); - } else { - throw new ServerApiException(ApiErrorCode.UNAUTHORIZED, - "Backroll API call unauthorized, please ask your administrator to fix integration issues."); - } - } - } catch (final IOException e) { - throw new CloudRuntimeException("Failed to authenticate Backroll API service due to:" + e.getMessage()); - } catch (JSONException e) { - e.printStackTrace(); - } - finally { - closeConnection(response); - } - logger.debug("Backroll client - end login"); - } - - private List getBackrollBackups(final String vmId) throws KeyManagementException, ParseException, NoSuchAlgorithmException { - try { - logger.info("start to list Backroll backups for vm {}", vmId); - - CloseableHttpResponse response = backrollService.get(apiURI,"/virtualmachines/" + vmId + "/backups"); - String result = backrollService.okBody(response); - BackrollTaskRequestResponse requestResponse = new ObjectMapper().readValue(result, BackrollTaskRequestResponse.class); - response.close(); - String urlToRequest = requestResponse.location.replace("/api/v1", ""); - - logger.debug(urlToRequest); - result = backrollService.waitGet(apiURI,urlToRequest); - BackrollBackupsFromVMResponse backrollBackupsFromVMResponse = new ObjectMapper().readValue(result, BackrollBackupsFromVMResponse.class); - - final List backups = new ArrayList<>(); - for (final BackrollArchiveResponse archive : backrollBackupsFromVMResponse.archives.archives) { - backups.add(new BackrollBackup(archive.name)); - logger.debug(archive.name); - } - return backups; - } catch (final NotOkBodyException e) { - throw new CloudRuntimeException("Failed to retrieve backups for this VM via Backroll"); - } catch (final IOException e) { - logger.error("Failed to list backup form vm with Backroll due to: {}", e); - } catch (InterruptedException e) { - logger.error("Backroll Error: {}", e); - } - return new ArrayList(); - } } \ No newline at end of file diff --git a/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/model/response/BackrollAsyncResponse.java b/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/model/response/BackrollAsyncResponse.java new file mode 100644 index 00000000000..84c94b5fbbc --- /dev/null +++ b/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/model/response/BackrollAsyncResponse.java @@ -0,0 +1,24 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.backup.backroll.model.response; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public abstract class BackrollAsyncResponse { + @JsonProperty("state") + public String state; +} diff --git a/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/model/response/archive/BackrollBackupsFromVMResponse.java b/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/model/response/archive/BackrollBackupsFromVMResponse.java index 6e066dce7f6..0fa3398bda8 100644 --- a/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/model/response/archive/BackrollBackupsFromVMResponse.java +++ b/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/model/response/archive/BackrollBackupsFromVMResponse.java @@ -1,27 +1,10 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. package org.apache.cloudstack.backup.backroll.model.response.archive; +import org.apache.cloudstack.backup.backroll.model.response.BackrollAsyncResponse; + import com.fasterxml.jackson.annotation.JsonProperty; -public class BackrollBackupsFromVMResponse { - @JsonProperty("state") - public String state; - +public class BackrollBackupsFromVMResponse extends BackrollAsyncResponse { @JsonProperty("info") public BackrollArchivesResponse archives; -} +} \ No newline at end of file diff --git a/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/model/response/backup/BackrollBackupResponse.java b/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/model/response/backup/BackrollBackupResponse.java new file mode 100644 index 00000000000..a8eb0b14f58 --- /dev/null +++ b/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/model/response/backup/BackrollBackupResponse.java @@ -0,0 +1,23 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.backup.backroll.model.response.backup; + +import org.apache.cloudstack.backup.backroll.model.response.BackrollAsyncResponse; + +public class BackrollBackupResponse extends BackrollAsyncResponse{ + +} \ No newline at end of file diff --git a/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/model/response/metrics/backup/BackrollBackupMetricsResponse.java b/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/model/response/metrics/backup/BackrollBackupMetricsResponse.java index f696ac5f20a..1582c0133f2 100644 --- a/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/model/response/metrics/backup/BackrollBackupMetricsResponse.java +++ b/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/model/response/metrics/backup/BackrollBackupMetricsResponse.java @@ -16,12 +16,11 @@ // under the License. package org.apache.cloudstack.backup.backroll.model.response.metrics.backup; +import org.apache.cloudstack.backup.backroll.model.response.BackrollAsyncResponse; + import com.fasterxml.jackson.annotation.JsonProperty; -public class BackrollBackupMetricsResponse { - @JsonProperty("state") - public String state; - +public class BackrollBackupMetricsResponse extends BackrollAsyncResponse { @JsonProperty("info") public BackupMetricsInfo info; } diff --git a/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/model/response/metrics/virtualMachine/BackrollVmMetricsResponse.java b/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/model/response/metrics/virtualMachine/BackrollVmMetricsResponse.java index 2a9fd490e7d..74bbca646ae 100644 --- a/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/model/response/metrics/virtualMachine/BackrollVmMetricsResponse.java +++ b/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/model/response/metrics/virtualMachine/BackrollVmMetricsResponse.java @@ -16,12 +16,11 @@ // under the License. package org.apache.cloudstack.backup.backroll.model.response.metrics.virtualMachine; +import org.apache.cloudstack.backup.backroll.model.response.BackrollAsyncResponse; + import com.fasterxml.jackson.annotation.JsonProperty; -public class BackrollVmMetricsResponse { - @JsonProperty("state") - public String state; - +public class BackrollVmMetricsResponse extends BackrollAsyncResponse{ @JsonProperty("info") public MetricsInfos infos; } diff --git a/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/model/response/metrics/virtualMachineBackups/VirtualMachineBackupsResponse.java b/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/model/response/metrics/virtualMachineBackups/VirtualMachineBackupsResponse.java index 5f70a91466c..11ed8267be6 100644 --- a/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/model/response/metrics/virtualMachineBackups/VirtualMachineBackupsResponse.java +++ b/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/model/response/metrics/virtualMachineBackups/VirtualMachineBackupsResponse.java @@ -16,12 +16,11 @@ // under the License. package org.apache.cloudstack.backup.backroll.model.response.metrics.virtualMachineBackups; +import org.apache.cloudstack.backup.backroll.model.response.BackrollAsyncResponse; + import com.fasterxml.jackson.annotation.JsonProperty; -public class VirtualMachineBackupsResponse { - @JsonProperty("state") - public String state; - +public class VirtualMachineBackupsResponse extends BackrollAsyncResponse { @JsonProperty("info") public Archives info; } diff --git a/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/model/response/policy/BackupPoliciesResponse.java b/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/model/response/policy/BackupPoliciesResponse.java index a125144b96d..f9512520386 100644 --- a/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/model/response/policy/BackupPoliciesResponse.java +++ b/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/model/response/policy/BackupPoliciesResponse.java @@ -18,12 +18,12 @@ package org.apache.cloudstack.backup.backroll.model.response.policy; import java.util.List; +import org.apache.cloudstack.backup.backroll.model.response.BackrollAsyncResponse; + import com.fasterxml.jackson.annotation.JsonFormat; import com.fasterxml.jackson.annotation.JsonProperty; -public class BackupPoliciesResponse { - @JsonProperty("state") - public String state; +public class BackupPoliciesResponse extends BackrollAsyncResponse { @JsonProperty("info") @JsonFormat(with = JsonFormat.Feature.ACCEPT_SINGLE_VALUE_AS_ARRAY) diff --git a/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/model/response/restore/BackrollRestoreResponse.java b/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/model/response/restore/BackrollRestoreResponse.java new file mode 100644 index 00000000000..8f831f49e2f --- /dev/null +++ b/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/model/response/restore/BackrollRestoreResponse.java @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.backup.backroll.model.response.restore; + +import org.apache.cloudstack.backup.backroll.model.response.BackrollAsyncResponse; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public class BackrollRestoreResponse extends BackrollAsyncResponse { + @JsonProperty("info") + public String info; +} diff --git a/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/utils/BackrollApiException.java b/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/utils/BackrollApiException.java new file mode 100644 index 00000000000..a07f9e9863d --- /dev/null +++ b/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/utils/BackrollApiException.java @@ -0,0 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.backup.backroll.utils; + + +public class BackrollApiException extends Exception { + +} diff --git a/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/utils/BackrollHttpClientProvider.java b/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/utils/BackrollHttpClientProvider.java new file mode 100644 index 00000000000..823e7ffb972 --- /dev/null +++ b/plugins/backup/backroll/src/main/java/org/apache/cloudstack/backup/backroll/utils/BackrollHttpClientProvider.java @@ -0,0 +1,395 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.backup.backroll.utils; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.util.concurrent.TimeUnit; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.X509TrustManager; + +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.backup.backroll.BackrollClient; +import org.apache.cloudstack.backup.backroll.model.response.TaskState; +import org.apache.cloudstack.backup.backroll.model.response.api.LoginApiResponse; +import org.apache.cloudstack.utils.security.SSLUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.http.HttpEntity; +import org.apache.http.HttpHeaders; +import org.apache.http.HttpStatus; +import org.apache.http.ParseException; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpDelete; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.util.EntityUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.json.JSONException; +import org.json.JSONObject; + +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.nio.TrustAllManager; +import com.fasterxml.jackson.databind.ObjectMapper; + +public class BackrollHttpClientProvider { + private final URI apiURI; + private String backrollToken = null; + private String appname = null; + private String password = null; + private RequestConfig config = null; + private boolean validateCertificate = false; + + private Logger logger = LogManager.getLogger(BackrollClient.class); + + public BackrollHttpClientProvider(final String url, final String appname, final String password, + final boolean validateCertificate, final int timeout, + final int restoreTimeout) + throws URISyntaxException, NoSuchAlgorithmException, KeyManagementException { + this.apiURI = new URI(url); + this.appname = appname; + this.password = password; + this.validateCertificate = validateCertificate; + + this.config = RequestConfig.custom() + .setConnectTimeout(timeout * 1000) + .setConnectionRequestTimeout(timeout * 1000) + .setSocketTimeout(timeout * 1000) + .build(); + } + + private CloseableHttpClient createHttpClient() throws BackrollApiException { + if(!validateCertificate) { + SSLContext sslContext; + try { + sslContext = SSLUtils.getSSLContext(); + sslContext.init(null, new X509TrustManager[] { new TrustAllManager() }, new SecureRandom()); + final SSLConnectionSocketFactory factory = new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE); + return HttpClientBuilder.create() + .setDefaultRequestConfig(config) + .setSSLSocketFactory(factory) + .build(); + } catch (NoSuchAlgorithmException | KeyManagementException e) { + logger.error(e); + e.printStackTrace(); + throw new BackrollApiException(); + } + } else { + return HttpClientBuilder.create() + .setDefaultRequestConfig(config) + .build(); + } + } + + public T post(final String path, final JSONObject json, Class classOfT) throws BackrollApiException, IOException { + + loginIfAuthenticationFailed(); + + try (CloseableHttpClient httpClient = createHttpClient()) { + + String xml = null; + StringEntity params = null; + if (json != null) { + logger.debug("JSON {}", json.toString()); + params = new StringEntity(json.toString(), ContentType.APPLICATION_JSON); + } + + String url = apiURI.toString() + path; + final HttpPost request = new HttpPost(url); + + if (params != null) { + request.setEntity(params); + } + + request.setHeader(HttpHeaders.AUTHORIZATION, "Bearer " + backrollToken); + request.setHeader("Content-type", "application/json"); + + final CloseableHttpResponse response = httpClient.execute(request); + + logger.debug("Response received in POST request with body {} is: {} for URL {}.", xml, response.toString(), url); + + String result = okBody(response); + T requestResponse = new ObjectMapper().readValue(result, classOfT); + response.close(); + + return requestResponse; + } catch (ParseException | NotOkBodyException e) { + logger.error(e); + e.printStackTrace(); + throw new BackrollApiException(); + } + } + + public T get(String path, Class classOfT) throws IOException, BackrollApiException{ + logger.debug("Backroll Get Auth "); + loginIfAuthenticationFailed(); + logger.debug("Backroll Get Auth ok"); + try (CloseableHttpClient httpClient = createHttpClient()) { + + String url = apiURI.toString() + path; + logger.debug("Backroll URL {}", url); + + HttpGet request = new HttpGet(url); + request.setHeader(HttpHeaders.AUTHORIZATION, "Bearer " + backrollToken); + request.setHeader("Content-type", "application/json"); + CloseableHttpResponse response = httpClient.execute(request); + logger.debug("Response received in GET request is: {} for URL: {}.", response.toString(), url); + + String result = okBody(response); + T requestResponse = new ObjectMapper().readValue(result, classOfT); + response.close(); + + return requestResponse; + } catch (NotOkBodyException e) { + logger.error(e); + e.printStackTrace(); + throw new BackrollApiException(); + } + } + + public String getWithoutParseResponse(String path) throws IOException, BackrollApiException{ + logger.debug("Backroll Get Auth "); + loginIfAuthenticationFailed(); + logger.debug("Backroll Get Auth ok"); + try (CloseableHttpClient httpClient = createHttpClient()) { + + String url = apiURI.toString() + path; + logger.debug("Backroll URL {}", url); + + HttpGet request = new HttpGet(url); + request.setHeader(HttpHeaders.AUTHORIZATION, "Bearer " + backrollToken); + request.setHeader("Content-type", "application/json"); + CloseableHttpResponse response = httpClient.execute(request); + logger.debug("Response received in GET request is: {} for URL: {}.", response.toString(), url); + + String result = okBody(response); + response.close(); + + return result; + } catch (NotOkBodyException e) { + logger.error(e); + e.printStackTrace(); + throw new BackrollApiException(); + } + } + + public T delete(String path, Class classOfT) throws IOException, BackrollApiException { + + loginIfAuthenticationFailed(); + + try (CloseableHttpClient httpClient = createHttpClient()) { + + String url = apiURI.toString() + path; + final HttpDelete request = new HttpDelete(url); + request.setHeader(HttpHeaders.AUTHORIZATION, "Bearer " + backrollToken); + request.setHeader("Content-type", "application/json"); + final CloseableHttpResponse response = httpClient.execute(request); + logger.debug("Response received in GET request is: {} for URL: {}.", response.toString(), url); + + String result = okBody(response); + T requestResponse = new ObjectMapper().readValue(result, classOfT); + response.close(); + + return requestResponse; + } catch (NotOkBodyException e) { + logger.error(e); + e.printStackTrace(); + throw new BackrollApiException(); + } + } + + public class NotOkBodyException extends Exception { + } + + public String okBody(final CloseableHttpResponse response) throws NotOkBodyException { + String result = ""; + switch (response.getStatusLine().getStatusCode()) { + case HttpStatus.SC_OK: + case HttpStatus.SC_ACCEPTED: + HttpEntity bodyEntity = response.getEntity(); + try { + result = EntityUtils.toString(bodyEntity); + EntityUtils.consumeQuietly(bodyEntity); + return result; + } catch (ParseException | IOException e) { + e.printStackTrace(); + throw new NotOkBodyException(); + } finally { + EntityUtils.consumeQuietly(bodyEntity); + } + default: + try { + closeConnection(response); + } catch (IOException e) { + e.printStackTrace(); + } + throw new NotOkBodyException(); + } + } + + public T waitGet(String url, Class classOfT) throws IOException, BackrollApiException { + loginIfAuthenticationFailed(); + + // int threshold = 30; // 5 minutes + int maxAttempts = 12; // 2 minutes + + for (int attempt = 1; attempt <= maxAttempts; attempt++) { + String body = getWithoutParseResponse(url); + if(!StringUtils.isEmpty(body)){ + if (!body.contains(TaskState.PENDING)) { + T result = new ObjectMapper().readValue(body, classOfT); + return result; + } + } + + try { + TimeUnit.SECONDS.sleep(10); + } catch (InterruptedException e) { + logger.error(e); + e.printStackTrace(); + throw new BackrollApiException(); + } + } + + return null; + } + + public String waitGetWithoutParseResponse(String url) + throws IOException, BackrollApiException { + // int threshold = 30; // 5 minutes + int maxAttempts = 12; // 2 minutes + + for (int attempt = 1; attempt <= maxAttempts; attempt++) { + + String body = getWithoutParseResponse(url); + if (!body.contains(TaskState.PENDING)) { + return body; + } + + try { + TimeUnit.SECONDS.sleep(10); + } catch (InterruptedException e) { + logger.error(e); + e.printStackTrace(); + throw new BackrollApiException(); + } + } + + return null; + } + + private boolean isAuthenticated() throws BackrollApiException, IOException { + boolean result = false; + + if(StringUtils.isEmpty(backrollToken)) { + logger.debug("Backroll Tocken empty : " + backrollToken); + return result; + } + + try (CloseableHttpClient httpClient = createHttpClient()) { + final HttpPost request = new HttpPost(apiURI.toString() + "/auth"); + CloseableHttpResponse httpResponse = httpClient.execute(request); + logger.debug("Backroll Auth response : " + EntityUtils.toString(httpResponse.getEntity())); + logger.debug("Backroll Auth response status : " + httpResponse.getStatusLine().getStatusCode()); + String response = okBody(httpResponse); + logger.debug("Backroll Auth response 2 : " + response); + logger.debug("Backroll Auth ok"); + result = true; + }catch (NotOkBodyException e) { + logger.error(e); + e.printStackTrace(); + result = false; + } + return result; + } + + private void closeConnection(CloseableHttpResponse closeableHttpResponse) throws IOException { + closeableHttpResponse.close(); + } + + public void loginIfAuthenticationFailed() throws BackrollApiException, IOException { + if (!isAuthenticated()) { + login(appname, password); + } + } + + private void login(final String appname, final String appsecret) throws BackrollApiException, IOException { + logger.debug("Backroll client - start login"); + + CloseableHttpClient httpClient = createHttpClient(); + CloseableHttpResponse httpResponse = null; + + final HttpPost request = new HttpPost(apiURI.toString() + "/login"); + request.addHeader("content-type", "application/json"); + + JSONObject jsonBody = new JSONObject(); + StringEntity params; + + try { + jsonBody.put("app_id", appname); + jsonBody.put("app_secret", appsecret); + params = new StringEntity(jsonBody.toString()); + request.setEntity(params); + + httpResponse = httpClient.execute(request); + try { + String response = okBody(httpResponse); + ObjectMapper objectMapper = new ObjectMapper(); + logger.info("BACKROLL: " + response); + LoginApiResponse loginResponse = objectMapper.readValue(response, LoginApiResponse.class); + logger.info("ok"); + this.backrollToken = loginResponse.accessToken; + logger.debug("Backroll client - Token : {}", backrollToken); + + if (StringUtils.isEmpty(loginResponse.accessToken)) { + throw new CloudRuntimeException("Backroll token is not available to perform API requests"); + } + } catch (final NotOkBodyException e) { + logger.error(e); + if (httpResponse.getStatusLine().getStatusCode() != HttpStatus.SC_CREATED) { + throw new CloudRuntimeException("Failed to create and authenticate Backroll client, please check the settings."); + } else { + throw new ServerApiException(ApiErrorCode.UNAUTHORIZED, "Backroll API call unauthorized, please ask your administrator to fix integration issues."); + } + } + } catch (final IOException e) { + logger.error(e); + throw new CloudRuntimeException("Failed to authenticate Backroll API service due to:" + e.getMessage()); + } catch (JSONException e) { + e.printStackTrace(); + logger.error(e); + throw new CloudRuntimeException("Failed to authenticate Backroll API service due to:" + e.getMessage()); + } + finally { + closeConnection(httpResponse); + } + logger.debug("Backroll client - end login"); + } +} diff --git a/plugins/backup/dummy/pom.xml b/plugins/backup/dummy/pom.xml index 52fbd085eb2..3a3749e50a8 100644 --- a/plugins/backup/dummy/pom.xml +++ b/plugins/backup/dummy/pom.xml @@ -23,7 +23,7 @@ cloudstack-plugins org.apache.cloudstack - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../../pom.xml diff --git a/plugins/backup/nas/pom.xml b/plugins/backup/nas/pom.xml index 096bf45c67e..8160826e573 100644 --- a/plugins/backup/nas/pom.xml +++ b/plugins/backup/nas/pom.xml @@ -25,7 +25,7 @@ cloudstack-plugins org.apache.cloudstack - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../../pom.xml diff --git a/plugins/backup/networker/pom.xml b/plugins/backup/networker/pom.xml index 1124d281d2e..fd1247ee58e 100644 --- a/plugins/backup/networker/pom.xml +++ b/plugins/backup/networker/pom.xml @@ -25,7 +25,7 @@ cloudstack-plugins org.apache.cloudstack - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../../pom.xml diff --git a/plugins/backup/veeam/pom.xml b/plugins/backup/veeam/pom.xml index a317f90388a..0e8f8c708a6 100644 --- a/plugins/backup/veeam/pom.xml +++ b/plugins/backup/veeam/pom.xml @@ -23,7 +23,7 @@ cloudstack-plugins org.apache.cloudstack - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../../pom.xml diff --git a/plugins/ca/root-ca/pom.xml b/plugins/ca/root-ca/pom.xml index fe1fb006302..da42fd9083c 100644 --- a/plugins/ca/root-ca/pom.xml +++ b/plugins/ca/root-ca/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../../pom.xml diff --git a/plugins/database/mysql-ha/pom.xml b/plugins/database/mysql-ha/pom.xml index 37a07784341..b23719df14f 100644 --- a/plugins/database/mysql-ha/pom.xml +++ b/plugins/database/mysql-ha/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../../pom.xml diff --git a/plugins/database/quota/pom.xml b/plugins/database/quota/pom.xml index b574b263020..3439f2f855c 100644 --- a/plugins/database/quota/pom.xml +++ b/plugins/database/quota/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../../pom.xml diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaValidateActivationRuleCmd.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaValidateActivationRuleCmd.java new file mode 100644 index 00000000000..a9dc7ea63eb --- /dev/null +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaValidateActivationRuleCmd.java @@ -0,0 +1,70 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//with the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. +package org.apache.cloudstack.api.command; + +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.user.Account; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.QuotaResponseBuilder; +import org.apache.cloudstack.api.response.QuotaValidateActivationRuleResponse; +import org.apache.cloudstack.quota.constant.QuotaTypes; + +import javax.inject.Inject; + +@APICommand(name = "quotaValidateActivationRule", responseObject = QuotaValidateActivationRuleResponse.class, description = "Validates if the given activation rule is valid for the informed usage type.", since = "4.20.0", requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) +public class QuotaValidateActivationRuleCmd extends BaseCmd { + + @Inject + QuotaResponseBuilder responseBuilder; + + @Parameter(name = ApiConstants.ACTIVATION_RULE, type = CommandType.STRING, required = true, description = "Quota tariff's activation rule to validate. The activation rule is valid if it has no syntax errors and all " + + "variables are compatible with the given usage type.", length = 65535) + private String activationRule; + + @Parameter(name = ApiConstants.USAGE_TYPE, type = CommandType.INTEGER, required = true, description = "The Quota usage type used to validate the activation rule.") + private Integer quotaType; + + @Override + public void execute() { + QuotaValidateActivationRuleResponse response = responseBuilder.validateActivationRule(this); + + response.setResponseName(getCommandName()); + setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + return Account.ACCOUNT_ID_SYSTEM; + } + + public String getActivationRule() { + return activationRule; + } + + public QuotaTypes getQuotaType() { + QuotaTypes quotaTypes = QuotaTypes.getQuotaType(quotaType); + + if (quotaTypes == null) { + throw new InvalidParameterValueException(String.format("Usage type not found for value [%s].", quotaType)); + } + + return quotaTypes; + } +} diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilder.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilder.java index c635551aeb5..56935a1360c 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilder.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilder.java @@ -26,6 +26,7 @@ import org.apache.cloudstack.api.command.QuotaStatementCmd; import org.apache.cloudstack.api.command.QuotaTariffCreateCmd; import org.apache.cloudstack.api.command.QuotaTariffListCmd; import org.apache.cloudstack.api.command.QuotaTariffUpdateCmd; +import org.apache.cloudstack.api.command.QuotaValidateActivationRuleCmd; import org.apache.cloudstack.quota.vo.QuotaBalanceVO; import org.apache.cloudstack.quota.vo.QuotaEmailConfigurationVO; import org.apache.cloudstack.quota.vo.QuotaTariffVO; @@ -88,4 +89,6 @@ public interface QuotaResponseBuilder { QuotaConfigureEmailResponse createQuotaConfigureEmailResponse(QuotaEmailConfigurationVO quotaEmailConfigurationVO, Double minBalance, long accountId); List listEmailConfiguration(long accountId); + + QuotaValidateActivationRuleResponse validateActivationRule(QuotaValidateActivationRuleCmd cmd); } diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java index 1c486759e43..733f7792356 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java @@ -16,6 +16,7 @@ //under the License. package org.apache.cloudstack.api.response; +import java.io.IOException; import java.lang.reflect.Field; import java.lang.reflect.Modifier; import java.lang.reflect.ParameterizedType; @@ -34,12 +35,15 @@ import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.ListIterator; +import java.util.Map; +import java.util.Set; import java.util.function.Consumer; import java.util.stream.Collectors; import javax.inject.Inject; import com.cloud.utils.DateUtil; +import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.command.QuotaBalanceCmd; @@ -51,8 +55,10 @@ import org.apache.cloudstack.api.command.QuotaStatementCmd; import org.apache.cloudstack.api.command.QuotaTariffCreateCmd; import org.apache.cloudstack.api.command.QuotaTariffListCmd; import org.apache.cloudstack.api.command.QuotaTariffUpdateCmd; +import org.apache.cloudstack.api.command.QuotaValidateActivationRuleCmd; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.discovery.ApiDiscoveryService; +import org.apache.cloudstack.jsinterpreter.JsInterpreterHelper; import org.apache.cloudstack.quota.QuotaManager; import org.apache.cloudstack.quota.QuotaManagerImpl; import org.apache.cloudstack.quota.QuotaService; @@ -78,6 +84,7 @@ import org.apache.cloudstack.quota.vo.QuotaEmailConfigurationVO; import org.apache.cloudstack.quota.vo.QuotaEmailTemplatesVO; import org.apache.cloudstack.quota.vo.QuotaTariffVO; import org.apache.cloudstack.quota.vo.QuotaUsageVO; +import org.apache.cloudstack.utils.jsinterpreter.JsInterpreter; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.reflect.FieldUtils; @@ -133,11 +140,13 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { private QuotaManager _quotaManager; @Inject private QuotaEmailConfigurationDao quotaEmailConfigurationDao; + @Inject + private JsInterpreterHelper jsInterpreterHelper; + @Inject + private ApiDiscoveryService apiDiscoveryService; private final Class[] assignableClasses = {GenericPresetVariable.class, ComputingResources.class}; - @Inject - private ApiDiscoveryService apiDiscoveryService; @Override public QuotaTariffResponse createQuotaTariffResponse(QuotaTariffVO tariff, boolean returnActivationRule) { @@ -789,7 +798,7 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { */ public void filterSupportedTypes(List> variables, QuotaTypes quotaType, PresetVariableDefinition presetVariableDefinitionAnnotation, Class fieldClass, String presetVariableName) { - if (Arrays.stream(presetVariableDefinitionAnnotation.supportedTypes()).noneMatch(supportedType -> + if (quotaType != null && Arrays.stream(presetVariableDefinitionAnnotation.supportedTypes()).noneMatch(supportedType -> supportedType == quotaType.getQuotaType() || supportedType == 0)) { return; } @@ -928,4 +937,82 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { return quotaConfigureEmailResponse; } + + @Override + public QuotaValidateActivationRuleResponse validateActivationRule(QuotaValidateActivationRuleCmd cmd) { + String message; + String activationRule = cmd.getActivationRule(); + QuotaTypes quotaType = cmd.getQuotaType(); + String quotaName = quotaType.getQuotaName(); + List> usageTypeVariablesAndDescriptions = new ArrayList<>(); + + addAllPresetVariables(PresetVariables.class, quotaType, usageTypeVariablesAndDescriptions, null); + List usageTypeVariables = usageTypeVariablesAndDescriptions.stream().map(Pair::first).collect(Collectors.toList()); + + try (JsInterpreter jsInterpreter = new JsInterpreter(QuotaConfig.QuotaActivationRuleTimeout.value())) { + Map newVariables = injectUsageTypeVariables(jsInterpreter, usageTypeVariables); + String scriptToExecute = jsInterpreterHelper.replaceScriptVariables(activationRule, newVariables); + jsInterpreter.executeScript(String.format("new Function(\"%s\")", scriptToExecute.replaceAll("\n", ""))); + } catch (IOException | CloudRuntimeException e) { + logger.error("Unable to execute activation rule due to: [{}].", e.getMessage(), e); + message = "Error while executing activation rule. Check if there are no syntax errors and all variables are compatible with the given usage type."; + return createValidateActivationRuleResponse(activationRule, quotaName, false, message); + } + + Set scriptVariables = jsInterpreterHelper.getScriptVariables(activationRule); + if (isScriptVariablesValid(scriptVariables, usageTypeVariables)) { + message = "The script has no syntax errors and all variables are compatible with the given usage type."; + return createValidateActivationRuleResponse(activationRule, quotaName, true, message); + } + + message = "Found variables that are not compatible with the given usage type."; + return createValidateActivationRuleResponse(activationRule, quotaName, false, message); + } + + /** + * Checks whether script variables are compatible with the usage type. First, we remove all script variables that correspond to the script's usage type variables. + * Then, returns true if none of the remaining script variables match any usage types variables, and false otherwise. + * + * @param scriptVariables Script variables. + * @param scriptUsageTypeVariables Script usage type variables. + * @return True if the script variables are valid, false otherwise. + */ + protected boolean isScriptVariablesValid(Set scriptVariables, List scriptUsageTypeVariables) { + List> allUsageTypeVariablesAndDescriptions = new ArrayList<>(); + addAllPresetVariables(PresetVariables.class, null, allUsageTypeVariablesAndDescriptions, null); + List allUsageTypesVariables = allUsageTypeVariablesAndDescriptions.stream().map(Pair::first).collect(Collectors.toList()); + + List matchVariables = scriptVariables.stream().filter(scriptUsageTypeVariables::contains).collect(Collectors.toList()); + matchVariables.forEach(scriptVariables::remove); + + return scriptVariables.stream().noneMatch(allUsageTypesVariables::contains); + } + + /** + * Injects variables into JavaScript interpreter. It's necessary to remove all dots from the given variables, as the interpreter + * does not interpret the variables as attributes of objects. + * + * @param jsInterpreter the {@link JsInterpreter} which the variables will be injected. + * @param variables the {@link List} with variables to format and inject the formatted variables into interpreter. + * @return A {@link Map} which has the key as the given variable and the value as the given variable formatted (without dots). + */ + protected Map injectUsageTypeVariables(JsInterpreter jsInterpreter, List variables) { + Map formattedVariables = new HashMap<>(); + for (String variable : variables) { + String formattedVariable = variable.replace(".", ""); + formattedVariables.put(variable, formattedVariable); + jsInterpreter.injectVariable(formattedVariable, "false"); + } + + return formattedVariables; + } + + public QuotaValidateActivationRuleResponse createValidateActivationRuleResponse(String activationRule, String quotaType, Boolean isValid, String message) { + QuotaValidateActivationRuleResponse response = new QuotaValidateActivationRuleResponse(); + response.setActivationRule(activationRule); + response.setQuotaType(quotaType); + response.setValid(isValid); + response.setMessage(message); + return response; + } } diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaValidateActivationRuleResponse.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaValidateActivationRuleResponse.java new file mode 100644 index 00000000000..0726764568f --- /dev/null +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaValidateActivationRuleResponse.java @@ -0,0 +1,76 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//with the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. +package org.apache.cloudstack.api.response; + +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; +import org.apache.cloudstack.api.BaseResponse; + +public class QuotaValidateActivationRuleResponse extends BaseResponse { + + @SerializedName("activationrule") + @Param(description = "The validated activation rule.") + private String activationRule; + + @SerializedName("quotatype") + @Param(description = "The Quota usage type used to validate the activation rule.") + private String quotaType; + + @SerializedName("isvalid") + @Param(description = "Whether the activation rule is valid.") + private Boolean isValid; + + @SerializedName("message") + @Param(description = "The reason whether the activation rule is valid or not.") + private String message; + + public QuotaValidateActivationRuleResponse() { + super("validactivationrule"); + } + + public String getActivationRule() { + return activationRule; + } + + public void setActivationRule(String activationRule) { + this.activationRule = activationRule; + } + + public Boolean isValid() { + return isValid; + } + + public void setValid(Boolean valid) { + isValid = valid; + } + + public String getQuotaType() { + return quotaType; + } + + public void setQuotaType(String quotaType) { + this.quotaType = quotaType; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } +} diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java index 17fa7bd8425..97d77b8aa22 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java @@ -41,6 +41,7 @@ import org.apache.cloudstack.api.command.QuotaTariffDeleteCmd; import org.apache.cloudstack.api.command.QuotaTariffListCmd; import org.apache.cloudstack.api.command.QuotaTariffUpdateCmd; import org.apache.cloudstack.api.command.QuotaUpdateCmd; +import org.apache.cloudstack.api.command.QuotaValidateActivationRuleCmd; import org.apache.cloudstack.api.response.QuotaResponseBuilder; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.framework.config.ConfigKey; @@ -121,6 +122,7 @@ public class QuotaServiceImpl extends ManagerBase implements QuotaService, Confi cmdList.add(QuotaConfigureEmailCmd.class); cmdList.add(QuotaListEmailConfigurationCmd.class); cmdList.add(QuotaPresetVariablesListCmd.class); + cmdList.add(QuotaValidateActivationRuleCmd.class); return cmdList; } diff --git a/plugins/database/quota/src/test/java/org/apache/cloudstack/api/command/QuotaValidateActivationRuleCmdTest.java b/plugins/database/quota/src/test/java/org/apache/cloudstack/api/command/QuotaValidateActivationRuleCmdTest.java new file mode 100644 index 00000000000..1d0c0cf7ff6 --- /dev/null +++ b/plugins/database/quota/src/test/java/org/apache/cloudstack/api/command/QuotaValidateActivationRuleCmdTest.java @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command; + +import org.apache.cloudstack.api.response.QuotaResponseBuilder; +import org.apache.cloudstack.api.response.QuotaValidateActivationRuleResponse; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class QuotaValidateActivationRuleCmdTest { + @Mock + QuotaResponseBuilder responseBuilderMock; + + @Test + public void executeTestVerifyCalls() { + QuotaValidateActivationRuleCmd cmd = new QuotaValidateActivationRuleCmd(); + cmd.responseBuilder = responseBuilderMock; + Mockito.doReturn(new QuotaValidateActivationRuleResponse()).when(responseBuilderMock).validateActivationRule(cmd); + cmd.execute(); + + Mockito.verify(responseBuilderMock).validateActivationRule(cmd); + } +} diff --git a/plugins/database/quota/src/test/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImplTest.java b/plugins/database/quota/src/test/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImplTest.java index fd359525893..a26e6c0476d 100644 --- a/plugins/database/quota/src/test/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImplTest.java +++ b/plugins/database/quota/src/test/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImplTest.java @@ -25,6 +25,9 @@ import java.util.ArrayList; import java.util.Calendar; import java.util.Date; import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.HashSet; import java.util.function.Consumer; import com.cloud.domain.DomainVO; @@ -34,7 +37,10 @@ import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.command.QuotaConfigureEmailCmd; import org.apache.cloudstack.api.command.QuotaEmailTemplateListCmd; import org.apache.cloudstack.api.command.QuotaEmailTemplateUpdateCmd; +import org.apache.cloudstack.api.command.QuotaValidateActivationRuleCmd; +import org.apache.cloudstack.discovery.ApiDiscoveryService; import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.jsinterpreter.JsInterpreterHelper; import org.apache.cloudstack.quota.QuotaService; import org.apache.cloudstack.quota.QuotaStatement; import org.apache.cloudstack.quota.activationrule.presetvariables.PresetVariableDefinition; @@ -55,7 +61,7 @@ import org.apache.cloudstack.quota.vo.QuotaCreditsVO; import org.apache.cloudstack.quota.vo.QuotaEmailConfigurationVO; import org.apache.cloudstack.quota.vo.QuotaEmailTemplatesVO; import org.apache.cloudstack.quota.vo.QuotaTariffVO; -import org.apache.cloudstack.discovery.ApiDiscoveryService; +import org.apache.cloudstack.utils.jsinterpreter.JsInterpreter; import org.apache.commons.lang3.time.DateUtils; @@ -160,6 +166,12 @@ public class QuotaResponseBuilderImplTest extends TestCase { return new Calendar[] {calendar, calendar}; } + @Mock + QuotaValidateActivationRuleCmd quotaValidateActivationRuleCmdMock = Mockito.mock(QuotaValidateActivationRuleCmd.class); + + @Mock + JsInterpreterHelper jsInterpreterHelperMock = Mockito.mock(JsInterpreterHelper.class); + private QuotaTariffVO makeTariffTestData() { QuotaTariffVO tariffVO = new QuotaTariffVO(); tariffVO.setUsageType(QuotaTypes.IP_ADDRESS); @@ -645,4 +657,78 @@ public class QuotaResponseBuilderImplTest extends TestCase { assertFalse(quotaResponseBuilderSpy.isUserAllowedToSeeActivationRules(userMock)); } + + @Test + public void validateActivationRuleTestValidateActivationRuleReturnValidScriptResponse() { + Mockito.doReturn("if (account.name == 'test') { true } else { false }").when(quotaValidateActivationRuleCmdMock).getActivationRule(); + Mockito.doReturn(QuotaTypes.getQuotaType(30)).when(quotaValidateActivationRuleCmdMock).getQuotaType(); + Mockito.doReturn(quotaValidateActivationRuleCmdMock.getActivationRule()).when(jsInterpreterHelperMock).replaceScriptVariables(Mockito.anyString(), Mockito.any()); + + QuotaValidateActivationRuleResponse response = quotaResponseBuilderSpy.validateActivationRule(quotaValidateActivationRuleCmdMock); + + Assert.assertTrue(response.isValid()); + } + + @Test + public void validateActivationRuleTestUsageTypeIncompatibleVariableReturnInvalidScriptResponse() { + Mockito.doReturn("if (value.osName == 'test') { true } else { false }").when(quotaValidateActivationRuleCmdMock).getActivationRule(); + Mockito.doReturn(QuotaTypes.getQuotaType(30)).when(quotaValidateActivationRuleCmdMock).getQuotaType(); + Mockito.doReturn(quotaValidateActivationRuleCmdMock.getActivationRule()).when(jsInterpreterHelperMock).replaceScriptVariables(Mockito.anyString(), Mockito.any()); + Mockito.when(jsInterpreterHelperMock.getScriptVariables(quotaValidateActivationRuleCmdMock.getActivationRule())).thenReturn(Set.of("value.osName")); + + QuotaValidateActivationRuleResponse response = quotaResponseBuilderSpy.validateActivationRule(quotaValidateActivationRuleCmdMock); + + Assert.assertFalse(response.isValid()); + } + + @Test + public void validateActivationRuleTestActivationRuleWithSyntaxErrorsReturnInvalidScriptResponse() { + Mockito.doReturn("{ if (account.name == 'test') { true } else { false } }}").when(quotaValidateActivationRuleCmdMock).getActivationRule(); + Mockito.doReturn(QuotaTypes.getQuotaType(1)).when(quotaValidateActivationRuleCmdMock).getQuotaType(); + Mockito.doReturn(quotaValidateActivationRuleCmdMock.getActivationRule()).when(jsInterpreterHelperMock).replaceScriptVariables(Mockito.anyString(), Mockito.any()); + + QuotaValidateActivationRuleResponse response = quotaResponseBuilderSpy.validateActivationRule(quotaValidateActivationRuleCmdMock); + + Assert.assertFalse(response.isValid()); + } + + @Test + public void isScriptVariablesValidTestUnsupportedUsageTypeVariablesReturnFalse() { + Set scriptVariables = new HashSet<>(List.of("value.computingResources.cpuNumber", "account.name", "zone.id")); + List usageTypeVariables = List.of("value.virtualSize", "account.name", "zone.id"); + + boolean isScriptVariablesValid = quotaResponseBuilderSpy.isScriptVariablesValid(scriptVariables, usageTypeVariables); + + Assert.assertFalse(isScriptVariablesValid); + } + + @Test + public void isScriptVariablesValidTestSupportedUsageTypeVariablesReturnTrue() { + Set scriptVariables = new HashSet<>(List.of("value.computingResources.cpuNumber", "account.name", "zone.id")); + List usageTypeVariables = List.of("value.computingResources.cpuNumber", "account.name", "zone.id"); + + boolean isScriptVariablesValid = quotaResponseBuilderSpy.isScriptVariablesValid(scriptVariables, usageTypeVariables); + + Assert.assertTrue(isScriptVariablesValid); + } + + @Test + public void isScriptVariablesValidTestVariablesUnrelatedToUsageTypeReturnTrue() { + Set scriptVariables = new HashSet<>(List.of("variable1.valid", "variable2.valid.", "variable3.valid")); + List usageTypeVariables = List.of("project.name", "account.id", "domain.path"); + + boolean isScriptVariablesValid = quotaResponseBuilderSpy.isScriptVariablesValid(scriptVariables, usageTypeVariables); + + Assert.assertTrue(isScriptVariablesValid); + } + + @Test + public void injectUsageTypeVariablesTestReturnInjectedVariables() { + JsInterpreter interpreter = Mockito.mock(JsInterpreter.class); + + Map formattedVariables = quotaResponseBuilderSpy.injectUsageTypeVariables(interpreter, List.of("account.name", "zone.name")); + + Assert.assertTrue(formattedVariables.containsValue("accountname")); + Assert.assertTrue(formattedVariables.containsValue("zonename")); + } } diff --git a/plugins/dedicated-resources/pom.xml b/plugins/dedicated-resources/pom.xml index 5aeecec818d..cffeddec7a7 100644 --- a/plugins/dedicated-resources/pom.xml +++ b/plugins/dedicated-resources/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../pom.xml diff --git a/plugins/deployment-planners/implicit-dedication/pom.xml b/plugins/deployment-planners/implicit-dedication/pom.xml index b9f8be5d08a..12454d37270 100644 --- a/plugins/deployment-planners/implicit-dedication/pom.xml +++ b/plugins/deployment-planners/implicit-dedication/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../../pom.xml diff --git a/plugins/deployment-planners/user-concentrated-pod/pom.xml b/plugins/deployment-planners/user-concentrated-pod/pom.xml index 0dcbe3506c6..ad3ac14a5b5 100644 --- a/plugins/deployment-planners/user-concentrated-pod/pom.xml +++ b/plugins/deployment-planners/user-concentrated-pod/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../../pom.xml diff --git a/plugins/deployment-planners/user-dispersing/pom.xml b/plugins/deployment-planners/user-dispersing/pom.xml index bbd74bf1a7a..a05577ed04b 100644 --- a/plugins/deployment-planners/user-dispersing/pom.xml +++ b/plugins/deployment-planners/user-dispersing/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../../pom.xml diff --git a/plugins/drs/cluster/balanced/pom.xml b/plugins/drs/cluster/balanced/pom.xml index 743a5f2bc98..4db1bd796f8 100644 --- a/plugins/drs/cluster/balanced/pom.xml +++ b/plugins/drs/cluster/balanced/pom.xml @@ -27,7 +27,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../../../pom.xml diff --git a/plugins/drs/cluster/condensed/pom.xml b/plugins/drs/cluster/condensed/pom.xml index 60b472bccb5..29a8ae2081d 100644 --- a/plugins/drs/cluster/condensed/pom.xml +++ b/plugins/drs/cluster/condensed/pom.xml @@ -27,7 +27,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../../../pom.xml diff --git a/plugins/event-bus/inmemory/pom.xml b/plugins/event-bus/inmemory/pom.xml index be85e8afd8d..d3b389d9b3f 100644 --- a/plugins/event-bus/inmemory/pom.xml +++ b/plugins/event-bus/inmemory/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../../pom.xml diff --git a/plugins/event-bus/kafka/pom.xml b/plugins/event-bus/kafka/pom.xml index 44014847548..95667c5cc79 100644 --- a/plugins/event-bus/kafka/pom.xml +++ b/plugins/event-bus/kafka/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../../pom.xml diff --git a/plugins/event-bus/rabbitmq/pom.xml b/plugins/event-bus/rabbitmq/pom.xml index 1e04caf026f..5ccc2a813ed 100644 --- a/plugins/event-bus/rabbitmq/pom.xml +++ b/plugins/event-bus/rabbitmq/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../../pom.xml diff --git a/plugins/event-bus/webhook/pom.xml b/plugins/event-bus/webhook/pom.xml index 278f4dc0ec5..e9dc24804f5 100644 --- a/plugins/event-bus/webhook/pom.xml +++ b/plugins/event-bus/webhook/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../../pom.xml diff --git a/plugins/ha-planners/skip-heurestics/pom.xml b/plugins/ha-planners/skip-heurestics/pom.xml index 3dc3989da2c..4f47bf7ee75 100644 --- a/plugins/ha-planners/skip-heurestics/pom.xml +++ b/plugins/ha-planners/skip-heurestics/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../../pom.xml diff --git a/plugins/host-allocators/random/pom.xml b/plugins/host-allocators/random/pom.xml index f01494914ce..8f04ec9b63b 100644 --- a/plugins/host-allocators/random/pom.xml +++ b/plugins/host-allocators/random/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../../pom.xml diff --git a/plugins/hypervisors/baremetal/pom.xml b/plugins/hypervisors/baremetal/pom.xml index d866c9b47e2..f75cb4a4b6c 100755 --- a/plugins/hypervisors/baremetal/pom.xml +++ b/plugins/hypervisors/baremetal/pom.xml @@ -22,7 +22,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../../pom.xml cloud-plugin-hypervisor-baremetal diff --git a/plugins/hypervisors/hyperv/pom.xml b/plugins/hypervisors/hyperv/pom.xml index 56b2b6d1503..76026b31845 100644 --- a/plugins/hypervisors/hyperv/pom.xml +++ b/plugins/hypervisors/hyperv/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../../pom.xml diff --git a/plugins/hypervisors/kvm/pom.xml b/plugins/hypervisors/kvm/pom.xml index f1eb4ea8c4f..08ad50ce2cb 100644 --- a/plugins/hypervisors/kvm/pom.xml +++ b/plugins/hypervisors/kvm/pom.xml @@ -24,7 +24,7 @@ org.apache.cloudstack cloudstack-plugins - 4.20.0.0-SNAPSHOT + 4.21.0.0-SNAPSHOT ../../pom.xml diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 0b0416a8049..7374d46edd6 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -3000,6 +3000,17 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv return dataPath; } + public static boolean useBLOCKDiskType(KVMPhysicalDisk physicalDisk) { + return physicalDisk != null && + physicalDisk.getPool().getType() == StoragePoolType.Linstor && + physicalDisk.getFormat() != null && + physicalDisk.getFormat()== PhysicalDiskFormat.RAW; + } + + public static DiskDef.DiskType getDiskType(KVMPhysicalDisk physicalDisk) { + return useBLOCKDiskType(physicalDisk) ? DiskDef.DiskType.BLOCK : DiskDef.DiskType.FILE; + } + public void createVbd(final Connect conn, final VirtualMachineTO vmSpec, final String vmName, final LibvirtVMDef vm) throws InternalErrorException, LibvirtException, URISyntaxException { final Map details = vmSpec.getDetails(); final List disks = Arrays.asList(vmSpec.getDisks()); @@ -3045,7 +3056,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv physicalDisk = getPhysicalDiskFromNfsStore(dataStoreUrl, data); } else if (primaryDataStoreTO.getPoolType().equals(StoragePoolType.SharedMountPoint) || primaryDataStoreTO.getPoolType().equals(StoragePoolType.Filesystem) || - primaryDataStoreTO.getPoolType().equals(StoragePoolType.StorPool)) { + primaryDataStoreTO.getPoolType().equals(StoragePoolType.StorPool) || + primaryDataStoreTO.getPoolType().equals(StoragePoolType.Linstor)) { physicalDisk = getPhysicalDiskPrimaryStore(primaryDataStoreTO, data); } } @@ -3095,8 +3107,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv final DiskDef disk = new DiskDef(); int devId = volume.getDiskSeq().intValue(); if (volume.getType() == Volume.Type.ISO) { - - disk.defISODisk(volPath, devId, isUefiEnabled); + final DiskDef.DiskType diskType = getDiskType(physicalDisk); + disk.defISODisk(volPath, devId, isUefiEnabled, diskType); if (guestCpuArch != null && guestCpuArch.equals("aarch64")) { disk.setBusType(DiskDef.DiskBus.SCSI); @@ -3195,7 +3207,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv if (vmSpec.getType() != VirtualMachine.Type.User) { final DiskDef iso = new DiskDef(); - iso.defISODisk(sysvmISOPath); + iso.defISODisk(sysvmISOPath, DiskDef.DiskType.FILE); if (guestCpuArch != null && guestCpuArch.equals("aarch64")) { iso.setBusType(DiskDef.DiskBus.SCSI); } @@ -3408,7 +3420,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv List disks = getDisks(conn, vmName); DiskDef configdrive = null; for (DiskDef disk : disks) { - if (disk.getDeviceType() == DiskDef.DeviceType.CDROM && disk.getDiskLabel() == CONFIG_DRIVE_ISO_DISK_LABEL) { + if (disk.getDeviceType() == DiskDef.DeviceType.CDROM && CONFIG_DRIVE_ISO_DISK_LABEL.equals(disk.getDiskLabel())) { configdrive = disk; } } @@ -3438,11 +3450,12 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv final String name = isoPath.substring(index + 1); final KVMStoragePool secondaryPool = storagePoolManager.getStoragePoolByURI(path); final KVMPhysicalDisk isoVol = secondaryPool.getPhysicalDisk(name); + final DiskDef.DiskType diskType = getDiskType(isoVol); isoPath = isoVol.getPath(); - iso.defISODisk(isoPath, diskSeq); + iso.defISODisk(isoPath, diskSeq, diskType); } else { - iso.defISODisk(null, diskSeq); + iso.defISODisk(null, diskSeq, DiskDef.DiskType.FILE); } final String result = attachOrDetachDevice(conn, true, vmName, iso.toString()); @@ -3450,7 +3463,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv final List disks = getDisks(conn, vmName); for (final DiskDef disk : disks) { if (disk.getDeviceType() == DiskDef.DeviceType.CDROM - && (diskSeq == null || disk.getDiskLabel() == iso.getDiskLabel())) { + && (diskSeq == null || disk.getDiskLabel().equals(iso.getDiskLabel()))) { cleanupDisk(disk); } } @@ -4046,7 +4059,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv return stopVMInternal(conn, vmName, true); } String ret = stopVMInternal(conn, vmName, false); - if (ret == Script.ERR_TIMEOUT) { + if (Script.ERR_TIMEOUT.equals(ret)) { ret = stopVMInternal(conn, vmName, true); } else if (ret != null) { /* diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtDomainXMLParser.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtDomainXMLParser.java index 46620e77fda..c4d5ac48f6f 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtDomainXMLParser.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtDomainXMLParser.java @@ -127,11 +127,10 @@ public class LibvirtDomainXMLParser { } def.defFileBasedDisk(diskFile, diskLabel, DiskDef.DiskBus.valueOf(bus.toUpperCase()), fmt); } else if (device.equalsIgnoreCase("cdrom")) { - def.defISODisk(diskFile, i+1, diskLabel); + def.defISODisk(diskFile, i+1, diskLabel, DiskDef.DiskType.FILE); } } else if (type.equalsIgnoreCase("block")) { - def.defBlockBasedDisk(diskDev, diskLabel, - DiskDef.DiskBus.valueOf(bus.toUpperCase())); + parseDiskBlock(def, device, diskDev, diskLabel, bus, diskFile, i); } if (StringUtils.isNotBlank(diskCacheMode)) { def.setCacheMode(DiskDef.DiskCacheMode.valueOf(diskCacheMode.toUpperCase())); @@ -450,6 +449,25 @@ public class LibvirtDomainXMLParser { return node.getAttribute(attr); } + /** + * Parse the disk block part of the libvirt XML. + * @param def + * @param device + * @param diskDev + * @param diskLabel + * @param bus + * @param diskFile + * @param curDiskIndex + */ + private void parseDiskBlock(DiskDef def, String device, String diskDev, String diskLabel, String bus, + String diskFile, int curDiskIndex) { + if (device.equalsIgnoreCase("disk")) { + def.defBlockBasedDisk(diskDev, diskLabel, DiskDef.DiskBus.valueOf(bus.toUpperCase())); + } else if (device.equalsIgnoreCase("cdrom")) { + def.defISODisk(diskFile, curDiskIndex+1, diskLabel, DiskDef.DiskType.BLOCK); + } + } + public Integer getVncPort() { return vncPort; } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolDef.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolDef.java index 09ee45d5908..4f8b2843ec2 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolDef.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtStoragePoolDef.java @@ -24,7 +24,7 @@ import org.apache.commons.collections.CollectionUtils; public class LibvirtStoragePoolDef { public enum PoolType { - ISCSI("iscsi"), NETFS("netfs"), loggerICAL("logical"), DIR("dir"), RBD("rbd"), GLUSTERFS("glusterfs"), POWERFLEX("powerflex"); + ISCSI("iscsi"), NETFS("netfs"), LOGICAL("logical"), DIR("dir"), RBD("rbd"), GLUSTERFS("glusterfs"), POWERFLEX("powerflex"); String _poolType; PoolType(String poolType) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java index 0f11c12f101..a67294ecadb 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java @@ -894,8 +894,8 @@ public class LibvirtVMDef { } } - public void defISODisk(String volPath) { - _diskType = DiskType.FILE; + public void defISODisk(String volPath, DiskType diskType) { + _diskType = diskType; _deviceType = DeviceType.CDROM; _sourcePath = volPath; _diskLabel = getDevLabel(3, DiskBus.IDE, true); @@ -904,8 +904,8 @@ public class LibvirtVMDef { _bus = DiskBus.IDE; } - public void defISODisk(String volPath, boolean isUefiEnabled) { - _diskType = DiskType.FILE; + public void defISODisk(String volPath, boolean isUefiEnabled, DiskType diskType) { + _diskType = diskType; _deviceType = DeviceType.CDROM; _sourcePath = volPath; _bus = isUefiEnabled ? DiskBus.SATA : DiskBus.IDE; @@ -914,18 +914,18 @@ public class LibvirtVMDef { _diskCacheMode = DiskCacheMode.NONE; } - public void defISODisk(String volPath, Integer devId) { - defISODisk(volPath, devId, null); + public void defISODisk(String volPath, Integer devId, DiskType diskType) { + defISODisk(volPath, devId, null, diskType); } - public void defISODisk(String volPath, Integer devId, String diskLabel) { + public void defISODisk(String volPath, Integer devId, String diskLabel, DiskType diskType) { if (devId == null && StringUtils.isBlank(diskLabel)) { LOGGER.debug(String.format("No ID or label informed for volume [%s].", volPath)); - defISODisk(volPath); + defISODisk(volPath, diskType); return; } - _diskType = DiskType.FILE; + _diskType = diskType; _deviceType = DeviceType.CDROM; _sourcePath = volPath; @@ -942,11 +942,11 @@ public class LibvirtVMDef { _bus = DiskBus.IDE; } - public void defISODisk(String volPath, Integer devId,boolean isSecure) { + public void defISODisk(String volPath, Integer devId, boolean isSecure, DiskType diskType) { if (!isSecure) { - defISODisk(volPath, devId); + defISODisk(volPath, devId, diskType); } else { - _diskType = DiskType.FILE; + _diskType = diskType; _deviceType = DeviceType.CDROM; _sourcePath = volPath; _diskLabel = getDevLabel(devId, DiskBus.SATA, true); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapper.java index cc955e86d8a..3ba1bc11975 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapper.java @@ -130,34 +130,25 @@ public class LibvirtConvertInstanceCommandWrapper extends CommandWrapper temporaryDisks = xmlParser == null ? - getTemporaryDisksWithPrefixFromTemporaryPool(temporaryStoragePool, temporaryConvertPath, temporaryConvertUuid) : - getTemporaryDisksFromParsedXml(temporaryStoragePool, xmlParser, convertedBasePath); - - List destinationDisks = moveTemporaryDisksToDestination(temporaryDisks, - destinationStoragePools, storagePoolMgr); - - cleanupDisksAndDomainFromTemporaryLocation(temporaryDisks, temporaryStoragePool, temporaryConvertUuid); - - UnmanagedInstanceTO convertedInstanceTO = getConvertedUnmanagedInstance(temporaryConvertUuid, - destinationDisks, xmlParser); - return new ConvertInstanceAnswer(cmd, convertedInstanceTO); + return new ConvertInstanceAnswer(cmd, temporaryConvertUuid); } catch (Exception e) { String error = String.format("Error converting instance %s from %s, due to: %s", sourceInstanceName, sourceHypervisorType, e.getMessage()); logger.error(error, e); + cleanupSecondaryStorage = true; return new ConvertInstanceAnswer(cmd, false, error); } finally { if (ovfExported && StringUtils.isNotBlank(ovfTemplateDirOnConversionLocation)) { @@ -165,7 +156,7 @@ public class LibvirtConvertInstanceCommandWrapper extends CommandWrapper { + + @Override + public Answer execute(ImportConvertedInstanceCommand cmd, LibvirtComputingResource serverResource) { + RemoteInstanceTO sourceInstance = cmd.getSourceInstance(); + Hypervisor.HypervisorType sourceHypervisorType = sourceInstance.getHypervisorType(); + String sourceInstanceName = sourceInstance.getInstanceName(); + List destinationStoragePools = cmd.getDestinationStoragePools(); + DataStoreTO conversionTemporaryLocation = cmd.getConversionTemporaryLocation(); + final String temporaryConvertUuid = cmd.getTemporaryConvertUuid(); + + final KVMStoragePoolManager storagePoolMgr = serverResource.getStoragePoolMgr(); + KVMStoragePool temporaryStoragePool = getTemporaryStoragePool(conversionTemporaryLocation, storagePoolMgr); + final String temporaryConvertPath = temporaryStoragePool.getLocalPath(); + + try { + String convertedBasePath = String.format("%s/%s", temporaryConvertPath, temporaryConvertUuid); + LibvirtDomainXMLParser xmlParser = parseMigratedVMXmlDomain(convertedBasePath); + + List temporaryDisks = xmlParser == null ? + getTemporaryDisksWithPrefixFromTemporaryPool(temporaryStoragePool, temporaryConvertPath, temporaryConvertUuid) : + getTemporaryDisksFromParsedXml(temporaryStoragePool, xmlParser, convertedBasePath); + + List destinationDisks = moveTemporaryDisksToDestination(temporaryDisks, + destinationStoragePools, storagePoolMgr); + + cleanupDisksAndDomainFromTemporaryLocation(temporaryDisks, temporaryStoragePool, temporaryConvertUuid); + + UnmanagedInstanceTO convertedInstanceTO = getConvertedUnmanagedInstance(temporaryConvertUuid, + destinationDisks, xmlParser); + return new ImportConvertedInstanceAnswer(cmd, convertedInstanceTO); + } catch (Exception e) { + String error = String.format("Error converting instance %s from %s, due to: %s", + sourceInstanceName, sourceHypervisorType, e.getMessage()); + logger.error(error, e); + return new ImportConvertedInstanceAnswer(cmd, false, error); + } finally { + if (conversionTemporaryLocation instanceof NfsTO) { + logger.debug("Cleaning up secondary storage temporary location"); + storagePoolMgr.deleteStoragePool(temporaryStoragePool.getType(), temporaryStoragePool.getUuid()); + } + } + } + + protected KVMStoragePool getTemporaryStoragePool(DataStoreTO conversionTemporaryLocation, KVMStoragePoolManager storagePoolMgr) { + if (conversionTemporaryLocation instanceof NfsTO) { + NfsTO nfsTO = (NfsTO) conversionTemporaryLocation; + return storagePoolMgr.getStoragePoolByURI(nfsTO.getUrl()); + } else { + PrimaryDataStoreTO primaryDataStoreTO = (PrimaryDataStoreTO) conversionTemporaryLocation; + return storagePoolMgr.getStoragePool(primaryDataStoreTO.getPoolType(), primaryDataStoreTO.getUuid()); + } + } + + protected List getTemporaryDisksFromParsedXml(KVMStoragePool pool, LibvirtDomainXMLParser xmlParser, String convertedBasePath) { + List disksDefs = xmlParser.getDisks(); + disksDefs = disksDefs.stream().filter(x -> x.getDiskType() == LibvirtVMDef.DiskDef.DiskType.FILE && + x.getDeviceType() == LibvirtVMDef.DiskDef.DeviceType.DISK).collect(Collectors.toList()); + if (CollectionUtils.isEmpty(disksDefs)) { + String err = String.format("Cannot find any disk defined on the converted XML domain %s.xml", convertedBasePath); + logger.error(err); + throw new CloudRuntimeException(err); + } + sanitizeDisksPath(disksDefs); + return getPhysicalDisksFromDefPaths(disksDefs, pool); + } + + private List getPhysicalDisksFromDefPaths(List disksDefs, KVMStoragePool pool) { + List disks = new ArrayList<>(); + for (LibvirtVMDef.DiskDef diskDef : disksDefs) { + KVMPhysicalDisk physicalDisk = pool.getPhysicalDisk(diskDef.getDiskPath()); + disks.add(physicalDisk); + } + return disks; + } + + protected List getTemporaryDisksWithPrefixFromTemporaryPool(KVMStoragePool pool, String path, String prefix) { + String msg = String.format("Could not parse correctly the converted XML domain, checking for disks on %s with prefix %s", path, prefix); + logger.info(msg); + pool.refresh(); + List disksWithPrefix = pool.listPhysicalDisks() + .stream() + .filter(x -> x.getName().startsWith(prefix) && !x.getName().endsWith(".xml")) + .collect(Collectors.toList()); + if (CollectionUtils.isEmpty(disksWithPrefix)) { + msg = String.format("Could not find any converted disk with prefix %s on temporary location %s", prefix, path); + logger.error(msg); + throw new CloudRuntimeException(msg); + } + return disksWithPrefix; + } + + private void cleanupDisksAndDomainFromTemporaryLocation(List disks, + KVMStoragePool temporaryStoragePool, + String temporaryConvertUuid) { + for (KVMPhysicalDisk disk : disks) { + logger.info(String.format("Cleaning up temporary disk %s after conversion from temporary location", disk.getName())); + temporaryStoragePool.deletePhysicalDisk(disk.getName(), Storage.ImageFormat.QCOW2); + } + logger.info(String.format("Cleaning up temporary domain %s after conversion from temporary location", temporaryConvertUuid)); + FileUtil.deleteFiles(temporaryStoragePool.getLocalPath(), temporaryConvertUuid, ".xml"); + } + + protected void sanitizeDisksPath(List disks) { + for (LibvirtVMDef.DiskDef disk : disks) { + String[] diskPathParts = disk.getDiskPath().split("/"); + String relativePath = diskPathParts[diskPathParts.length - 1]; + disk.setDiskPath(relativePath); + } + } + + protected List moveTemporaryDisksToDestination(List temporaryDisks, + List destinationStoragePools, + KVMStoragePoolManager storagePoolMgr) { + List targetDisks = new ArrayList<>(); + if (temporaryDisks.size() != destinationStoragePools.size()) { + String warn = String.format("Discrepancy between the converted instance disks (%s) " + + "and the expected number of disks (%s)", temporaryDisks.size(), destinationStoragePools.size()); + logger.warn(warn); + } + for (int i = 0; i < temporaryDisks.size(); i++) { + String poolPath = destinationStoragePools.get(i); + KVMStoragePool destinationPool = storagePoolMgr.getStoragePool(Storage.StoragePoolType.NetworkFilesystem, poolPath); + if (destinationPool == null) { + String err = String.format("Could not find a storage pool by URI: %s", poolPath); + logger.error(err); + continue; + } + if (destinationPool.getType() != Storage.StoragePoolType.NetworkFilesystem) { + String err = String.format("Storage pool by URI: %s is not an NFS storage", poolPath); + logger.error(err); + continue; + } + KVMPhysicalDisk sourceDisk = temporaryDisks.get(i); + if (logger.isDebugEnabled()) { + String msg = String.format("Trying to copy converted instance disk number %s from the temporary location %s" + + " to destination storage pool %s", i, sourceDisk.getPool().getLocalPath(), destinationPool.getUuid()); + logger.debug(msg); + } + + String destinationName = UUID.randomUUID().toString(); + + KVMPhysicalDisk destinationDisk = storagePoolMgr.copyPhysicalDisk(sourceDisk, destinationName, destinationPool, 7200 * 1000); + targetDisks.add(destinationDisk); + } + return targetDisks; + } + + private UnmanagedInstanceTO getConvertedUnmanagedInstance(String baseName, + List vmDisks, + LibvirtDomainXMLParser xmlParser) { + UnmanagedInstanceTO instanceTO = new UnmanagedInstanceTO(); + instanceTO.setName(baseName); + instanceTO.setDisks(getUnmanagedInstanceDisks(vmDisks, xmlParser)); + instanceTO.setNics(getUnmanagedInstanceNics(xmlParser)); + return instanceTO; + } + + private List getUnmanagedInstanceNics(LibvirtDomainXMLParser xmlParser) { + List nics = new ArrayList<>(); + if (xmlParser != null) { + List interfaces = xmlParser.getInterfaces(); + for (LibvirtVMDef.InterfaceDef interfaceDef : interfaces) { + UnmanagedInstanceTO.Nic nic = new UnmanagedInstanceTO.Nic(); + nic.setMacAddress(interfaceDef.getMacAddress()); + nic.setNicId(interfaceDef.getBrName()); + nic.setAdapterType(interfaceDef.getModel().toString()); + nics.add(nic); + } + } + return nics; + } + + protected List getUnmanagedInstanceDisks(List vmDisks, LibvirtDomainXMLParser xmlParser) { + List instanceDisks = new ArrayList<>(); + List diskDefs = xmlParser != null ? xmlParser.getDisks() : null; + for (int i = 0; i< vmDisks.size(); i++) { + KVMPhysicalDisk physicalDisk = vmDisks.get(i); + KVMStoragePool storagePool = physicalDisk.getPool(); + UnmanagedInstanceTO.Disk disk = new UnmanagedInstanceTO.Disk(); + disk.setPosition(i); + Pair storagePoolHostAndPath = getNfsStoragePoolHostAndPath(storagePool); + disk.setDatastoreHost(storagePoolHostAndPath.first()); + disk.setDatastorePath(storagePoolHostAndPath.second()); + disk.setDatastoreName(storagePool.getUuid()); + disk.setDatastoreType(storagePool.getType().name()); + disk.setCapacity(physicalDisk.getVirtualSize()); + disk.setFileBaseName(physicalDisk.getName()); + if (CollectionUtils.isNotEmpty(diskDefs)) { + LibvirtVMDef.DiskDef diskDef = diskDefs.get(i); + disk.setController(diskDef.getBusType() != null ? diskDef.getBusType().toString() : LibvirtVMDef.DiskDef.DiskBus.VIRTIO.toString()); + } else { + // If the job is finished but we cannot parse the XML, the guest VM can use the virtio driver + disk.setController(LibvirtVMDef.DiskDef.DiskBus.VIRTIO.toString()); + } + instanceDisks.add(disk); + } + return instanceDisks; + } + + protected Pair getNfsStoragePoolHostAndPath(KVMStoragePool storagePool) { + String sourceHostIp = null; + String sourcePath = null; + List commands = new ArrayList<>(); + commands.add(new String[]{Script.getExecutableAbsolutePath("mount")}); + commands.add(new String[]{Script.getExecutableAbsolutePath("grep"), storagePool.getLocalPath()}); + String storagePoolMountPoint = Script.executePipedCommands(commands, 0).second(); + logger.debug(String.format("NFS Storage pool: %s - local path: %s, mount point: %s", storagePool.getUuid(), storagePool.getLocalPath(), storagePoolMountPoint)); + if (StringUtils.isNotEmpty(storagePoolMountPoint)) { + String[] res = storagePoolMountPoint.strip().split(" "); + res = res[0].split(":"); + if (res.length > 1) { + sourceHostIp = res[0].strip(); + sourcePath = res[1].strip(); + } + } + return new Pair<>(sourceHostIp, sourcePath); + } + + protected LibvirtDomainXMLParser parseMigratedVMXmlDomain(String installPath) throws IOException { + String xmlPath = String.format("%s.xml", installPath); + if (!new File(xmlPath).exists()) { + String err = String.format("Conversion failed. Unable to find the converted XML domain, expected %s", xmlPath); + logger.error(err); + throw new CloudRuntimeException(err); + } + InputStream is = new BufferedInputStream(new FileInputStream(xmlPath)); + String xml = IOUtils.toString(is, Charset.defaultCharset()); + final LibvirtDomainXMLParser parser = new LibvirtDomainXMLParser(); + try { + parser.parseDomainXML(xml); + return parser; + } catch (RuntimeException e) { + String err = String.format("Error parsing the converted instance XML domain at %s: %s", xmlPath, e.getMessage()); + logger.error(err, e); + logger.debug(xml); + return null; + } + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/IscsiAdmStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/IscsiAdmStorageAdaptor.java index f023457461e..09cdad64ace 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/IscsiAdmStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/IscsiAdmStorageAdaptor.java @@ -43,7 +43,7 @@ public class IscsiAdmStorageAdaptor implements StorageAdaptor { private static final Map MapStorageUuidToStoragePool = new HashMap<>(); @Override - public KVMStoragePool createStoragePool(String uuid, String host, int port, String path, String userInfo, StoragePoolType storagePoolType, Map details) { + public KVMStoragePool createStoragePool(String uuid, String host, int port, String path, String userInfo, StoragePoolType storagePoolType, Map details, boolean isPrimaryStorage) { IscsiAdmStoragePool storagePool = new IscsiAdmStoragePool(uuid, host, port, storagePoolType, this); MapStorageUuidToStoragePool.put(uuid, storagePool); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java index 3c8026c7ffd..4e76030d06f 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java @@ -389,7 +389,7 @@ public class KVMStoragePoolManager { //Note: due to bug CLOUDSTACK-4459, createStoragepool can be called in parallel, so need to be synced. private synchronized KVMStoragePool createStoragePool(String name, String host, int port, String path, String userInfo, StoragePoolType type, Map details, boolean primaryStorage) { StorageAdaptor adaptor = getStorageAdaptor(type); - KVMStoragePool pool = adaptor.createStoragePool(name, host, port, path, userInfo, type, details); + KVMStoragePool pool = adaptor.createStoragePool(name, host, port, path, userInfo, type, details, primaryStorage); // LibvirtStorageAdaptor-specific statement if (pool.isPoolSupportHA() && primaryStorage) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index 391c7674552..fe6be10f9b8 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -26,9 +26,11 @@ import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; import java.text.DateFormat; import java.text.SimpleDateFormat; +import java.util.ArrayList; import java.util.Arrays; import java.util.Date; import java.util.HashMap; @@ -63,15 +65,19 @@ import org.apache.cloudstack.storage.command.ResignatureCommand; import org.apache.cloudstack.storage.command.SnapshotAndCopyAnswer; import org.apache.cloudstack.storage.command.SnapshotAndCopyCommand; import org.apache.cloudstack.storage.command.SyncVolumePathCommand; +import org.apache.cloudstack.storage.formatinspector.Qcow2Inspector; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; import org.apache.cloudstack.storage.to.SnapshotObjectTO; import org.apache.cloudstack.storage.to.TemplateObjectTO; import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.apache.cloudstack.utils.cryptsetup.KeyFile; +import org.apache.cloudstack.utils.qemu.QemuImageOptions; import org.apache.cloudstack.utils.qemu.QemuImg; import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; import org.apache.cloudstack.utils.qemu.QemuImgException; import org.apache.cloudstack.utils.qemu.QemuImgFile; import org.apache.cloudstack.utils.qemu.QemuObject; +import org.apache.cloudstack.utils.qemu.QemuObject.EncryptFormat; import org.apache.commons.collections.MapUtils; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.BooleanUtils; @@ -80,6 +86,7 @@ import org.apache.commons.lang3.builder.ToStringBuilder; import org.apache.commons.lang3.builder.ToStringStyle; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; + import org.libvirt.Connect; import org.libvirt.Domain; import org.libvirt.DomainInfo; @@ -134,10 +141,7 @@ import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.Script; import com.cloud.utils.storage.S3.S3Utils; import com.cloud.vm.VmDetailConstants; -import org.apache.cloudstack.utils.cryptsetup.KeyFile; -import org.apache.cloudstack.utils.qemu.QemuImageOptions; -import org.apache.cloudstack.utils.qemu.QemuObject.EncryptFormat; -import java.util.ArrayList; + public class KVMStorageProcessor implements StorageProcessor { protected Logger logger = LogManager.getLogger(getClass()); @@ -1115,11 +1119,12 @@ public class KVMStorageProcessor implements StorageProcessor { storagePool = storagePoolMgr.getStoragePoolByURI(path); } final KVMPhysicalDisk isoVol = storagePool.getPhysicalDisk(name); + final DiskDef.DiskType isoDiskType = LibvirtComputingResource.getDiskType(isoVol); isoPath = isoVol.getPath(); - iso.defISODisk(isoPath, isUefiEnabled); + iso.defISODisk(isoPath, isUefiEnabled, isoDiskType); } else { - iso.defISODisk(null, isUefiEnabled); + iso.defISODisk(null, isUefiEnabled, DiskDef.DiskType.FILE); } final List disks = resource.getDisks(conn, vmName); @@ -2452,6 +2457,22 @@ public class KVMStorageProcessor implements StorageProcessor { template = storagePoolMgr.createPhysicalDiskFromDirectDownloadTemplate(tempFilePath, destTemplatePath, destPool, cmd.getFormat(), cmd.getWaitInMillSeconds()); + String templatePath = template.getPath(); + if (templatePath != null) { + try { + Qcow2Inspector.validateQcow2File(templatePath); + } catch (RuntimeException e) { + try { + Files.deleteIfExists(Path.of(templatePath)); + } catch (IOException ioException) { + logger.warn("Unable to remove file [{}]; consider removing it manually.", templatePath, ioException); + } + + logger.error("The downloaded file [{}] is not a valid QCOW2.", templatePath, e); + return new DirectDownloadAnswer(false, "The downloaded file is not a valid QCOW2. Ask the administrator to check the logs for more details.", true); + } + } + if (!storagePoolMgr.disconnectPhysicalDisk(pool.getPoolType(), pool.getUuid(), destTemplatePath)) { logger.warn("Unable to disconnect physical disk at path: " + destTemplatePath + ", in storage pool id: " + pool.getUuid()); } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java index 45a65037340..1ce901d857f 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java @@ -73,6 +73,7 @@ import static com.cloud.utils.NumbersUtil.toHumanReadableSize; import java.util.Arrays; import java.util.HashSet; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; @@ -81,6 +82,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { private StorageLayer _storageLayer; private String _mountPoint = "/mnt"; private String _manageSnapshotPath; + private static final ConcurrentHashMap storagePoolRefCounts = new ConcurrentHashMap<>(); private String rbdTemplateSnapName = "cloudstack-base-snap"; private static final int RBD_FEATURE_LAYERING = 1; @@ -171,7 +173,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { * Checks if downloaded template is extractable * @return true if it should be extracted, false if not */ - private boolean isTemplateExtractable(String templatePath) { + public static boolean isTemplateExtractable(String templatePath) { String type = Script.runSimpleBashScript("file " + templatePath + " | awk -F' ' '{print $2}'"); return type.equalsIgnoreCase("bzip2") || type.equalsIgnoreCase("gzip") || type.equalsIgnoreCase("zip"); } @@ -181,7 +183,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { * @param downloadedTemplateFile * @param templateUuid */ - private String getExtractCommandForDownloadedFile(String downloadedTemplateFile, String templateUuid) { + public static String getExtractCommandForDownloadedFile(String downloadedTemplateFile, String templateUuid) { if (downloadedTemplateFile.endsWith(".zip")) { return "unzip -p " + downloadedTemplateFile + " | cat > " + templateUuid; } else if (downloadedTemplateFile.endsWith(".bz2")) { @@ -196,7 +198,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { /** * Extract downloaded template into installPath, remove compressed file */ - private void extractDownloadedTemplate(String downloadedTemplateFile, KVMStoragePool destPool, String destinationFile) { + public static void extractDownloadedTemplate(String downloadedTemplateFile, KVMStoragePool destPool, String destinationFile) { String extractCommand = getExtractCommandForDownloadedFile(downloadedTemplateFile, destinationFile); Script.runSimpleBashScript(extractCommand); Script.runSimpleBashScript("rm -f " + downloadedTemplateFile); @@ -350,7 +352,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { String volgroupName = path; volgroupName = volgroupName.replaceFirst("/", ""); - LibvirtStoragePoolDef spd = new LibvirtStoragePoolDef(PoolType.loggerICAL, volgroupName, uuid, host, volgroupPath, volgroupPath); + LibvirtStoragePoolDef spd = new LibvirtStoragePoolDef(PoolType.LOGICAL, volgroupName, uuid, host, volgroupPath, volgroupPath); StoragePool sp = null; try { logger.debug(spd.toString()); @@ -542,7 +544,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { type = StoragePoolType.Filesystem; } else if (spd.getPoolType() == LibvirtStoragePoolDef.PoolType.RBD) { type = StoragePoolType.RBD; - } else if (spd.getPoolType() == LibvirtStoragePoolDef.PoolType.loggerICAL) { + } else if (spd.getPoolType() == LibvirtStoragePoolDef.PoolType.LOGICAL) { type = StoragePoolType.CLVM; } else if (spd.getPoolType() == LibvirtStoragePoolDef.PoolType.GLUSTERFS) { type = StoragePoolType.Gluster; @@ -644,8 +646,44 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { } } + /** + * adjust refcount + */ + private int adjustStoragePoolRefCount(String uuid, int adjustment) { + final String mutexKey = storagePoolRefCounts.keySet().stream() + .filter(k -> k.equals(uuid)) + .findFirst() + .orElse(uuid); + synchronized (mutexKey) { + // some access on the storagePoolRefCounts.key(mutexKey) element + int refCount = storagePoolRefCounts.computeIfAbsent(mutexKey, k -> 0); + refCount += adjustment; + if (refCount < 1) { + storagePoolRefCounts.remove(mutexKey); + } else { + storagePoolRefCounts.put(mutexKey, refCount); + } + return refCount; + } + } + /** + * Thread-safe increment storage pool usage refcount + * @param uuid UUID of the storage pool to increment the count + */ + private void incStoragePoolRefCount(String uuid) { + adjustStoragePoolRefCount(uuid, 1); + } + /** + * Thread-safe decrement storage pool usage refcount for the given uuid and return if storage pool still in use. + * @param uuid UUID of the storage pool to decrement the count + * @return true if the storage pool is still used, else false. + */ + private boolean decStoragePoolRefCount(String uuid) { + return adjustStoragePoolRefCount(uuid, -1) > 0; + } + @Override - public KVMStoragePool createStoragePool(String name, String host, int port, String path, String userInfo, StoragePoolType type, Map details) { + public KVMStoragePool createStoragePool(String name, String host, int port, String path, String userInfo, StoragePoolType type, Map details, boolean isPrimaryStorage) { logger.info("Attempting to create storage pool " + name + " (" + type.toString() + ") in libvirt"); StoragePool sp = null; @@ -751,6 +789,12 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { } try { + if (!isPrimaryStorage) { + // only ref count storage pools for secondary storage, as primary storage is assumed + // to be always mounted, as long the primary storage isn't fully deleted. + incStoragePoolRefCount(name); + } + if (sp.isActive() == 0) { logger.debug("Attempting to activate pool " + name); sp.create(0); @@ -762,6 +806,7 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { return getStoragePool(name); } catch (LibvirtException e) { + decStoragePoolRefCount(name); String error = e.toString(); if (error.contains("Storage source conflict")) { throw new CloudRuntimeException("A pool matching this location already exists in libvirt, " + @@ -812,6 +857,13 @@ public class LibvirtStorageAdaptor implements StorageAdaptor { @Override public boolean deleteStoragePool(String uuid) { logger.info("Attempting to remove storage pool " + uuid + " from libvirt"); + + // decrement and check if storage pool still in use + if (decStoragePoolRefCount(uuid)) { + logger.info(String.format("deleteStoragePool: Storage pool %s still in use", uuid)); + return true; + } + Connect conn = null; try { conn = LibvirtConnection.getConnection(); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ManagedNfsStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ManagedNfsStorageAdaptor.java index 8ec56b885f2..2304e1cbac5 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ManagedNfsStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ManagedNfsStorageAdaptor.java @@ -56,7 +56,7 @@ public class ManagedNfsStorageAdaptor implements StorageAdaptor { } @Override - public KVMStoragePool createStoragePool(String uuid, String host, int port, String path, String userInfo, StoragePoolType storagePoolType, Map details) { + public KVMStoragePool createStoragePool(String uuid, String host, int port, String path, String userInfo, StoragePoolType storagePoolType, Map details, boolean isPrimaryStorage) { LibvirtStoragePool storagePool = new LibvirtStoragePool(uuid, path, StoragePoolType.ManagedNFS, this, null); storagePool.setSourceHost(host); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIAdapterBase.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIAdapterBase.java index 03acfcc89ad..7a827d9afc3 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIAdapterBase.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIAdapterBase.java @@ -169,7 +169,7 @@ public abstract class MultipathSCSIAdapterBase implements StorageAdaptor { } @Override - public KVMStoragePool createStoragePool(String uuid, String host, int port, String path, String userInfo, Storage.StoragePoolType type, Map details) { + public KVMStoragePool createStoragePool(String uuid, String host, int port, String path, String userInfo, Storage.StoragePoolType type, Map details, boolean isPrimaryStorage) { LOGGER.info(String.format("createStoragePool(uuid,host,port,path,type) called with args (%s, %s, %s, %s, %s)", uuid, host, ""+port, path, type)); MultipathSCSIPool storagePool = new MultipathSCSIPool(uuid, host, port, path, type, details, this); MapStorageUuidToStoragePool.put(uuid, storagePool); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java index b33f49404ad..7477d768e9a 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java @@ -146,7 +146,7 @@ public class ScaleIOStorageAdaptor implements StorageAdaptor { } @Override - public KVMStoragePool createStoragePool(String uuid, String host, int port, String path, String userInfo, Storage.StoragePoolType type, Map details) { + public KVMStoragePool createStoragePool(String uuid, String host, int port, String path, String userInfo, Storage.StoragePoolType type, Map details, boolean isPrimaryStorage) { ScaleIOStoragePool storagePool = new ScaleIOStoragePool(uuid, host, port, path, type, details, this); MapStorageUuidToStoragePool.put(uuid, storagePool); return storagePool; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/StorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/StorageAdaptor.java index 9a27d44ff92..15b03a55a3e 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/StorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/StorageAdaptor.java @@ -40,7 +40,7 @@ public interface StorageAdaptor { // it with info from local disk, and return it public KVMPhysicalDisk getPhysicalDisk(String volumeUuid, KVMStoragePool pool); - public KVMStoragePool createStoragePool(String name, String host, int port, String path, String userInfo, StoragePoolType type, Map details); + public KVMStoragePool createStoragePool(String name, String host, int port, String path, String userInfo, StoragePoolType type, Map details, boolean isPrimaryStorage); public boolean deleteStoragePool(String uuid); diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapperTest.java index f0e94e59485..9537469145f 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapperTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapperTest.java @@ -18,7 +18,6 @@ // package com.cloud.hypervisor.kvm.resource.wrapper; -import java.util.Arrays; import java.util.List; import java.util.UUID; @@ -80,7 +79,6 @@ public class LibvirtConvertInstanceCommandWrapperTest { Mockito.when(storagePoolManager.getStoragePoolByURI(secondaryPoolUrl)).thenReturn(temporaryPool); KVMPhysicalDisk physicalDisk1 = Mockito.mock(KVMPhysicalDisk.class); KVMPhysicalDisk physicalDisk2 = Mockito.mock(KVMPhysicalDisk.class); - Mockito.when(temporaryPool.listPhysicalDisks()).thenReturn(Arrays.asList(physicalDisk1, physicalDisk2)); } @Test @@ -107,51 +105,6 @@ public class LibvirtConvertInstanceCommandWrapperTest { Assert.assertNotNull(temporaryStoragePool); } - @Test - public void testGetTemporaryDisksWithPrefixFromTemporaryPool() { - String convertPath = "/xyz"; - String convertPrefix = UUID.randomUUID().toString(); - KVMPhysicalDisk physicalDisk1 = Mockito.mock(KVMPhysicalDisk.class); - Mockito.when(physicalDisk1.getName()).thenReturn("disk1"); - KVMPhysicalDisk physicalDisk2 = Mockito.mock(KVMPhysicalDisk.class); - Mockito.when(physicalDisk2.getName()).thenReturn("disk2"); - - KVMPhysicalDisk convertedDisk1 = Mockito.mock(KVMPhysicalDisk.class); - Mockito.when(convertedDisk1.getName()).thenReturn(String.format("%s-sda", convertPrefix)); - KVMPhysicalDisk convertedDisk2 = Mockito.mock(KVMPhysicalDisk.class); - Mockito.when(convertedDisk2.getName()).thenReturn(String.format("%s-sdb", convertPrefix)); - KVMPhysicalDisk convertedXml = Mockito.mock(KVMPhysicalDisk.class); - Mockito.when(convertedXml.getName()).thenReturn(String.format("%s.xml", convertPrefix)); - Mockito.when(temporaryPool.listPhysicalDisks()).thenReturn(Arrays.asList(physicalDisk1, physicalDisk2, - convertedDisk1, convertedDisk2, convertedXml)); - - List convertedDisks = convertInstanceCommandWrapper.getTemporaryDisksWithPrefixFromTemporaryPool(temporaryPool, convertPath, convertPrefix); - Assert.assertEquals(2, convertedDisks.size()); - } - - @Test - public void testGetTemporaryDisksFromParsedXml() { - String relativePath = UUID.randomUUID().toString(); - String fullPath = String.format("/mnt/xyz/%s", relativePath); - - LibvirtVMDef.DiskDef diskDef = new LibvirtVMDef.DiskDef(); - LibvirtVMDef.DiskDef.DiskBus bus = LibvirtVMDef.DiskDef.DiskBus.VIRTIO; - LibvirtVMDef.DiskDef.DiskFmtType type = LibvirtVMDef.DiskDef.DiskFmtType.QCOW2; - diskDef.defFileBasedDisk(fullPath, "test", bus, type); - - LibvirtDomainXMLParser parser = Mockito.mock(LibvirtDomainXMLParser.class); - Mockito.when(parser.getDisks()).thenReturn(List.of(diskDef)); - - KVMPhysicalDisk convertedDisk1 = Mockito.mock(KVMPhysicalDisk.class); - Mockito.when(convertedDisk1.getName()).thenReturn("disk1"); - Mockito.when(temporaryPool.getPhysicalDisk(relativePath)).thenReturn(convertedDisk1); - - List disks = convertInstanceCommandWrapper.getTemporaryDisksFromParsedXml(temporaryPool, parser, ""); - Mockito.verify(convertInstanceCommandWrapper).sanitizeDisksPath(List.of(diskDef)); - Assert.assertEquals(1, disks.size()); - Assert.assertEquals("disk1", disks.get(0).getName()); - } - @Test public void testSanitizeDisksPath() { String relativePath = UUID.randomUUID().toString(); diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtImportConvertedInstanceCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtImportConvertedInstanceCommandWrapperTest.java new file mode 100644 index 00000000000..343a15b367d --- /dev/null +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtImportConvertedInstanceCommandWrapperTest.java @@ -0,0 +1,245 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.ImportConvertedInstanceCommand; +import com.cloud.agent.api.to.NfsTO; +import com.cloud.agent.api.to.RemoteInstanceTO; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.resource.LibvirtDomainXMLParser; +import com.cloud.hypervisor.kvm.resource.LibvirtVMDef; +import com.cloud.hypervisor.kvm.storage.KVMPhysicalDisk; +import com.cloud.hypervisor.kvm.storage.KVMStoragePool; +import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; +import com.cloud.storage.Storage; +import com.cloud.utils.Pair; +import com.cloud.utils.script.Script; +import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; +import org.apache.cloudstack.vm.UnmanagedInstanceTO; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.Arrays; +import java.util.List; +import java.util.UUID; + +@RunWith(MockitoJUnitRunner.class) +public class LibvirtImportConvertedInstanceCommandWrapperTest { + + @Spy + private LibvirtImportConvertedInstanceCommandWrapper importInstanceCommandWrapper = + Mockito.spy(LibvirtImportConvertedInstanceCommandWrapper.class); + + @Mock + private LibvirtComputingResource libvirtComputingResourceMock; + + @Mock + private KVMStoragePool temporaryPool; + @Mock + private KVMStoragePool destinationPool; + @Mock + private PrimaryDataStoreTO primaryDataStore; + @Mock + private NfsTO secondaryDataStore; + @Mock + private KVMStoragePoolManager storagePoolManager; + + private static final String secondaryPoolUrl = "nfs://192.168.1.1/secondary"; + private static final String vmName = "VmToImport"; + + @Before + public void setUp() { + Mockito.when(secondaryDataStore.getUrl()).thenReturn(secondaryPoolUrl); + Mockito.when(libvirtComputingResourceMock.getStoragePoolMgr()).thenReturn(storagePoolManager); + Mockito.when(storagePoolManager.getStoragePoolByURI(secondaryPoolUrl)).thenReturn(temporaryPool); + KVMPhysicalDisk physicalDisk1 = Mockito.mock(KVMPhysicalDisk.class); + KVMPhysicalDisk physicalDisk2 = Mockito.mock(KVMPhysicalDisk.class); + Mockito.when(temporaryPool.listPhysicalDisks()).thenReturn(Arrays.asList(physicalDisk1, physicalDisk2)); + } + + @Test + public void testGetTemporaryStoragePool() { + KVMStoragePool temporaryStoragePool = importInstanceCommandWrapper.getTemporaryStoragePool(secondaryDataStore, libvirtComputingResourceMock.getStoragePoolMgr()); + Assert.assertNotNull(temporaryStoragePool); + } + + @Test + public void testGetTemporaryDisksWithPrefixFromTemporaryPool() { + String convertPath = "/xyz"; + String convertPrefix = UUID.randomUUID().toString(); + KVMPhysicalDisk physicalDisk1 = Mockito.mock(KVMPhysicalDisk.class); + Mockito.when(physicalDisk1.getName()).thenReturn("disk1"); + KVMPhysicalDisk physicalDisk2 = Mockito.mock(KVMPhysicalDisk.class); + Mockito.when(physicalDisk2.getName()).thenReturn("disk2"); + + KVMPhysicalDisk convertedDisk1 = Mockito.mock(KVMPhysicalDisk.class); + Mockito.when(convertedDisk1.getName()).thenReturn(String.format("%s-sda", convertPrefix)); + KVMPhysicalDisk convertedDisk2 = Mockito.mock(KVMPhysicalDisk.class); + Mockito.when(convertedDisk2.getName()).thenReturn(String.format("%s-sdb", convertPrefix)); + KVMPhysicalDisk convertedXml = Mockito.mock(KVMPhysicalDisk.class); + Mockito.when(convertedXml.getName()).thenReturn(String.format("%s.xml", convertPrefix)); + Mockito.when(temporaryPool.listPhysicalDisks()).thenReturn(Arrays.asList(physicalDisk1, physicalDisk2, + convertedDisk1, convertedDisk2, convertedXml)); + + List convertedDisks = importInstanceCommandWrapper.getTemporaryDisksWithPrefixFromTemporaryPool(temporaryPool, convertPath, convertPrefix); + Assert.assertEquals(2, convertedDisks.size()); + } + + @Test + public void testGetTemporaryDisksFromParsedXml() { + String relativePath = UUID.randomUUID().toString(); + String fullPath = String.format("/mnt/xyz/%s", relativePath); + + LibvirtVMDef.DiskDef diskDef = new LibvirtVMDef.DiskDef(); + LibvirtVMDef.DiskDef.DiskBus bus = LibvirtVMDef.DiskDef.DiskBus.VIRTIO; + LibvirtVMDef.DiskDef.DiskFmtType type = LibvirtVMDef.DiskDef.DiskFmtType.QCOW2; + diskDef.defFileBasedDisk(fullPath, "test", bus, type); + + LibvirtDomainXMLParser parser = Mockito.mock(LibvirtDomainXMLParser.class); + Mockito.when(parser.getDisks()).thenReturn(List.of(diskDef)); + + KVMPhysicalDisk convertedDisk1 = Mockito.mock(KVMPhysicalDisk.class); + Mockito.when(convertedDisk1.getName()).thenReturn("disk1"); + Mockito.when(temporaryPool.getPhysicalDisk(relativePath)).thenReturn(convertedDisk1); + + List disks = importInstanceCommandWrapper.getTemporaryDisksFromParsedXml(temporaryPool, parser, ""); + Mockito.verify(importInstanceCommandWrapper).sanitizeDisksPath(List.of(diskDef)); + Assert.assertEquals(1, disks.size()); + Assert.assertEquals("disk1", disks.get(0).getName()); + } + + @Test + public void testSanitizeDisksPath() { + String relativePath = UUID.randomUUID().toString(); + String fullPath = String.format("/mnt/xyz/%s", relativePath); + LibvirtVMDef.DiskDef diskDef = new LibvirtVMDef.DiskDef(); + LibvirtVMDef.DiskDef.DiskBus bus = LibvirtVMDef.DiskDef.DiskBus.VIRTIO; + LibvirtVMDef.DiskDef.DiskFmtType type = LibvirtVMDef.DiskDef.DiskFmtType.QCOW2; + diskDef.defFileBasedDisk(fullPath, "test", bus, type); + + importInstanceCommandWrapper.sanitizeDisksPath(List.of(diskDef)); + Assert.assertEquals(relativePath, diskDef.getDiskPath()); + } + + @Test + public void testMoveTemporaryDisksToDestination() { + KVMPhysicalDisk sourceDisk = Mockito.mock(KVMPhysicalDisk.class); + Mockito.lenient().when(sourceDisk.getPool()).thenReturn(temporaryPool); + List disks = List.of(sourceDisk); + String destinationPoolUuid = UUID.randomUUID().toString(); + List destinationPools = List.of(destinationPoolUuid); + + KVMPhysicalDisk destDisk = Mockito.mock(KVMPhysicalDisk.class); + Mockito.when(destDisk.getPath()).thenReturn("xyz"); + Mockito.when(storagePoolManager.getStoragePool(Storage.StoragePoolType.NetworkFilesystem, destinationPoolUuid)) + .thenReturn(destinationPool); + Mockito.when(destinationPool.getType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem); + Mockito.when(storagePoolManager.copyPhysicalDisk(Mockito.eq(sourceDisk), Mockito.anyString(), Mockito.eq(destinationPool), Mockito.anyInt())) + .thenReturn(destDisk); + + List movedDisks = importInstanceCommandWrapper.moveTemporaryDisksToDestination(disks, destinationPools, storagePoolManager); + Assert.assertEquals(1, movedDisks.size()); + Assert.assertEquals("xyz", movedDisks.get(0).getPath()); + } + + @Test + public void testGetUnmanagedInstanceDisks() { + try (MockedStatic + diff --git a/ui/src/views/iam/EditUser.vue b/ui/src/views/iam/EditUser.vue index e082fd17a06..18f22b6f1e8 100644 --- a/ui/src/views/iam/EditUser.vue +++ b/ui/src/views/iam/EditUser.vue @@ -81,6 +81,16 @@ + + + + Enabled + Inherit + Disabled + +
{{ $t('label.cancel') }} {{ $t('label.ok') }} @@ -128,6 +138,11 @@ export default { this.initForm() this.fetchData() }, + computed: { + isRootAdmin () { + return this.$store.getters.userInfo?.roletype === 'Admin' + } + }, methods: { initForm () { this.formRef = ref() @@ -187,7 +202,8 @@ export default { username: values.username, email: values.email, firstname: values.firstname, - lastname: values.lastname + lastname: values.lastname, + apikeyaccess: values.apikeyaccess } if (this.isValidValueForKey(values, 'timezone') && values.timezone.length > 0) { params.timezone = values.timezone diff --git a/ui/src/views/infra/ManagementServerPeerTab.vue b/ui/src/views/infra/ManagementServerPeerTab.vue new file mode 100644 index 00000000000..47cffca2eee --- /dev/null +++ b/ui/src/views/infra/ManagementServerPeerTab.vue @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + + + diff --git a/ui/src/views/network/CreateIpv4SubnetForNetwork.vue b/ui/src/views/network/CreateIpv4SubnetForNetwork.vue index 44af8246066..af1ef8c0e25 100644 --- a/ui/src/views/network/CreateIpv4SubnetForNetwork.vue +++ b/ui/src/views/network/CreateIpv4SubnetForNetwork.vue @@ -150,6 +150,7 @@ export default { const params = { showicon: true } api('listZones', params).then(json => { this.zones = json.listzonesresponse.zone || [] + this.zones = this.zones.filter(zone => zone.routedmodeenabled) this.form.zoneid = this.zones[0].id || '' this.fetchParentSubnets(this.form.zoneid) }).finally(() => { diff --git a/ui/src/views/network/CreateIsolatedNetworkForm.vue b/ui/src/views/network/CreateIsolatedNetworkForm.vue index b60223219e7..2c3fa17fa35 100644 --- a/ui/src/views/network/CreateIsolatedNetworkForm.vue +++ b/ui/src/views/network/CreateIsolatedNetworkForm.vue @@ -552,8 +552,9 @@ export default { this.selectedNetworkOffering = {} api('listNetworkOfferings', params).then(json => { this.networkOfferings = json.listnetworkofferingsresponse.networkoffering - if (this.selectedZone.isnsxenabled) { - this.networkOfferings = this.networkOfferings.filter(offering => offering.fornsx) + this.networkOfferings = this.networkOfferings.filter(offering => offering.fornsx === this.selectedZone.isnsxenabled) + if (!this.selectedZone.routedmodeenabled) { + this.networkOfferings = this.networkOfferings.filter(offering => offering.networkmode !== 'ROUTED') } }).catch(error => { this.$notifyError(error) diff --git a/ui/src/views/network/CreateVpc.vue b/ui/src/views/network/CreateVpc.vue index b0a56ca724a..6fb9a712262 100644 --- a/ui/src/views/network/CreateVpc.vue +++ b/ui/src/views/network/CreateVpc.vue @@ -345,6 +345,10 @@ export default { this.loadingOffering = true api('listVPCOfferings', { zoneid: this.form.zoneid, state: 'Enabled' }).then((response) => { this.vpcOfferings = response.listvpcofferingsresponse.vpcoffering + this.vpcOfferings = this.vpcOfferings.filter(offering => offering.fornsx === this.selectedZone.isnsxenabled) + if (!this.selectedZone.routedmodeenabled) { + this.vpcOfferings = this.vpcOfferings.filter(offering => offering.networkmode !== 'ROUTED') + } this.form.vpcofferingid = this.vpcOfferings[0].id || '' this.selectedVpcOffering = this.vpcOfferings[0] || {} }).finally(() => { diff --git a/ui/src/views/network/LoadBalancing.vue b/ui/src/views/network/LoadBalancing.vue index eeec232cee5..09f9a39c4cf 100644 --- a/ui/src/views/network/LoadBalancing.vue +++ b/ui/src/views/network/LoadBalancing.vue @@ -37,7 +37,7 @@
- +
@@ -126,7 +126,7 @@ :rowKey="record => record.id">