diff --git a/agent/src/main/java/com/cloud/agent/resource/consoleproxy/ConsoleProxyResource.java b/agent/src/main/java/com/cloud/agent/resource/consoleproxy/ConsoleProxyResource.java index b0b1e487a26..83b11418f2c 100644 --- a/agent/src/main/java/com/cloud/agent/resource/consoleproxy/ConsoleProxyResource.java +++ b/agent/src/main/java/com/cloud/agent/resource/consoleproxy/ConsoleProxyResource.java @@ -331,7 +331,7 @@ public class ConsoleProxyResource extends ServerResourceBase implements ServerRe final Object resource = this; logger.info("Building class loader for com.cloud.consoleproxy.ConsoleProxy"); if (consoleProxyMain == null) { - logger.info("Running com.cloud.consoleproxy.ConsoleProxy with encryptor password={}", encryptorPassword); + logger.info("Running com.cloud.consoleproxy.ConsoleProxy"); consoleProxyMain = new Thread(new ManagedContextRunnable() { @Override protected void runInContext() { diff --git a/api/src/main/java/com/cloud/network/NetworkService.java b/api/src/main/java/com/cloud/network/NetworkService.java index 9b570fee5c5..cb72346678d 100644 --- a/api/src/main/java/com/cloud/network/NetworkService.java +++ b/api/src/main/java/com/cloud/network/NetworkService.java @@ -107,6 +107,10 @@ public interface NetworkService { PhysicalNetwork physicalNetwork, long zoneId, ControlledEntity.ACLType aclType) throws InsufficientCapacityException, ConcurrentOperationException, ResourceAllocationException; + Network createGuestNetwork(long networkOfferingId, String name, String displayText, Account owner, + PhysicalNetwork physicalNetwork, long zoneId, ControlledEntity.ACLType aclType, Pair vrIfaceMTUs) throws + InsufficientCapacityException, ConcurrentOperationException, ResourceAllocationException; + Pair, Integer> searchForNetworks(ListNetworksCmd cmd); boolean deleteNetwork(long networkId, boolean forced); diff --git a/api/src/main/java/org/apache/cloudstack/alert/AlertService.java b/api/src/main/java/org/apache/cloudstack/alert/AlertService.java index 1250284b5c2..4ae6288efce 100644 --- a/api/src/main/java/org/apache/cloudstack/alert/AlertService.java +++ b/api/src/main/java/org/apache/cloudstack/alert/AlertService.java @@ -71,8 +71,8 @@ public interface AlertService { public static final AlertType ALERT_TYPE_HA_ACTION = new AlertType((short)30, "ALERT.HA.ACTION", true); public static final AlertType ALERT_TYPE_CA_CERT = new AlertType((short)31, "ALERT.CA.CERT", true); public static final AlertType ALERT_TYPE_VM_SNAPSHOT = new AlertType((short)32, "ALERT.VM.SNAPSHOT", true); - public static final AlertType ALERT_TYPE_VR_PUBLIC_IFACE_MTU = new AlertType((short)32, "ALERT.VR.PUBLIC.IFACE.MTU", true); - public static final AlertType ALERT_TYPE_VR_PRIVATE_IFACE_MTU = new AlertType((short)32, "ALERT.VR.PRIVATE.IFACE.MTU", true); + public static final AlertType ALERT_TYPE_VR_PUBLIC_IFACE_MTU = new AlertType((short)33, "ALERT.VR.PUBLIC.IFACE.MTU", true); + public static final AlertType ALERT_TYPE_VR_PRIVATE_IFACE_MTU = new AlertType((short)34, "ALERT.VR.PRIVATE.IFACE.MTU", true); public short getType() { return type; diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 4abc0d13d74..daf1bdc705d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -1097,6 +1097,7 @@ public class ApiConstants { public static final String DOCKER_REGISTRY_EMAIL = "dockerregistryemail"; public static final String ISO_NAME = "isoname"; public static final String ISO_STATE = "isostate"; + public static final String ISO_URL = "isourl"; public static final String SEMANTIC_VERSION = "semanticversion"; public static final String KUBERNETES_VERSION_ID = "kubernetesversionid"; public static final String KUBERNETES_VERSION_NAME = "kubernetesversionname"; diff --git a/api/src/main/java/org/apache/cloudstack/api/BaseUpdateTemplateOrIsoCmd.java b/api/src/main/java/org/apache/cloudstack/api/BaseUpdateTemplateOrIsoCmd.java index 38cf765dd1a..696a500860e 100644 --- a/api/src/main/java/org/apache/cloudstack/api/BaseUpdateTemplateOrIsoCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/BaseUpdateTemplateOrIsoCmd.java @@ -42,7 +42,7 @@ public abstract class BaseUpdateTemplateOrIsoCmd extends BaseCmd { @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = TemplateResponse.class, required = true, description = "The ID of the image file") private Long id; - @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, description = "The name of the image file") + @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, length = 251, description = "The name of the image file") private String templateName; @Parameter(name = ApiConstants.OS_TYPE_ID, diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/AddSecondaryStorageCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/AddSecondaryStorageCmd.java index 9a7eff7e2e5..585fd1b87a8 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/AddSecondaryStorageCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/AddSecondaryStorageCmd.java @@ -29,6 +29,11 @@ import org.apache.cloudstack.api.response.ZoneResponse; import com.cloud.exception.DiscoveryException; import com.cloud.storage.ImageStore; import com.cloud.user.Account; +import org.apache.commons.collections.MapUtils; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; @APICommand(name = "addSecondaryStorage", description = "Adds secondary storage.", responseObject = ImageStoreResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) @@ -44,6 +49,9 @@ public class AddSecondaryStorageCmd extends BaseCmd { @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.UUID, entityType = ZoneResponse.class, description = "The Zone ID for the secondary storage") protected Long zoneId; + @Parameter(name = ApiConstants.DETAILS, type = CommandType.MAP, description = "Details in key/value pairs using format details[i].keyname=keyvalue. Example: details[0].copytemplatesfromothersecondarystorages=true") + protected Map details; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -56,6 +64,20 @@ public class AddSecondaryStorageCmd extends BaseCmd { return zoneId; } + public Map getDetails() { + Map detailsMap = new HashMap<>(); + if (MapUtils.isNotEmpty(details)) { + Collection props = details.values(); + for (Object prop : props) { + HashMap detail = (HashMap) prop; + for (Map.Entry entry: detail.entrySet()) { + detailsMap.put(entry.getKey(),entry.getValue()); + } + } + } + return detailsMap; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @@ -68,7 +90,7 @@ public class AddSecondaryStorageCmd extends BaseCmd { @Override public void execute(){ try{ - ImageStore result = _storageService.discoverImageStore(null, getUrl(), "NFS", getZoneId(), null); + ImageStore result = _storageService.discoverImageStore(null, getUrl(), "NFS", getZoneId(), getDetails()); ImageStoreResponse storeResponse = null; if (result != null ) { storeResponse = _responseGenerator.createImageStoreResponse(result); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java index a73a0d00ffb..972a9c8950c 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java @@ -109,6 +109,9 @@ public class ListHostsCmd extends BaseListCmd { @Parameter(name = ApiConstants.ARCH, type = CommandType.STRING, description = "CPU Arch of the host", since = "4.20.1") private String arch; + @Parameter(name = ApiConstants.VERSION, type = CommandType.STRING, description = "the host version", since = "4.20.3") + private String version; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -197,6 +200,10 @@ public class ListHostsCmd extends BaseListCmd { return StringUtils.isBlank(arch) ? null : CPU.CPUArch.fromType(arch); } + public String getVersion() { + return version; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/management/ListMgmtsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/management/ListMgmtsCmd.java index ecc59f7cab8..293cb34e702 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/management/ListMgmtsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/management/ListMgmtsCmd.java @@ -45,6 +45,10 @@ public class ListMgmtsCmd extends BaseListCmd { since = "4.20.1.0") private Boolean peers; + @Parameter(name = ApiConstants.VERSION, type = CommandType.STRING, + description = "the version of the management server", since = "4.20.3") + private String version; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -61,6 +65,10 @@ public class ListMgmtsCmd extends BaseListCmd { return BooleanUtils.toBooleanDefaultIfNull(peers, false); } + public String getVersion() { + return version; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/network/UpdateNetworkOfferingCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/network/UpdateNetworkOfferingCmd.java index 9af10262b2d..8910966ba2e 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/network/UpdateNetworkOfferingCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/network/UpdateNetworkOfferingCmd.java @@ -78,6 +78,7 @@ public class UpdateNetworkOfferingCmd extends BaseCmd { @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.STRING, + length = 4096, description = "The ID of the containing domain(s) as comma separated string, public for public offerings") private String domainIds; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/UpdateDiskOfferingCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/UpdateDiskOfferingCmd.java index 2f07f85f983..c93b5d41a1c 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/UpdateDiskOfferingCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/UpdateDiskOfferingCmd.java @@ -75,6 +75,7 @@ public class UpdateDiskOfferingCmd extends BaseCmd { @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.STRING, description = "The ID of the containing zone(s) as comma separated string, all for all zones offerings", + length = 4096, since = "4.13") private String zoneIds; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/UpdateServiceOfferingCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/UpdateServiceOfferingCmd.java index 0dc97659b9d..26c7d87ab45 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/UpdateServiceOfferingCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/UpdateServiceOfferingCmd.java @@ -69,6 +69,7 @@ public class UpdateServiceOfferingCmd extends BaseCmd { @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.STRING, description = "The ID of the containing zone(s) as comma separated string, all for all zones offerings", + length = 4096, since = "4.13") private String zoneIds; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vpc/UpdateVPCOfferingCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vpc/UpdateVPCOfferingCmd.java index b8a8077b30b..44bc88c8daf 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vpc/UpdateVPCOfferingCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vpc/UpdateVPCOfferingCmd.java @@ -65,6 +65,7 @@ public class UpdateVPCOfferingCmd extends BaseAsyncCmd { @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.STRING, description = "The ID of the containing zone(s) as comma separated string, all for all zones offerings", + length = 4096, since = "4.13") private String zoneIds; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/iso/RegisterIsoCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/iso/RegisterIsoCmd.java index f499c01ce58..2de0f96f271 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/iso/RegisterIsoCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/iso/RegisterIsoCmd.java @@ -70,7 +70,7 @@ public class RegisterIsoCmd extends BaseCmd implements UserCmd { @Parameter(name = ApiConstants.IS_EXTRACTABLE, type = CommandType.BOOLEAN, description = "True if the ISO or its derivatives are extractable; default is false") private Boolean extractable; - @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, required = true, description = "The name of the ISO") + @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, required = true, length = 251, description = "The name of the ISO") private String isoName; @Parameter(name = ApiConstants.OS_TYPE_ID, diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java index bd541b69183..078d4517f95 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java @@ -244,8 +244,7 @@ public class CreateSnapshotCmd extends BaseAsyncCreateCmd { } private Snapshot.LocationType getLocationType() { - - if (Snapshot.LocationType.values() == null || Snapshot.LocationType.values().length == 0 || locationType == null) { + if (locationType == null) { return null; } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVnfApplianceCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVnfApplianceCmd.java index 4d50dd9c39b..92ddfd5b235 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVnfApplianceCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVnfApplianceCmd.java @@ -43,7 +43,7 @@ import java.util.List; public class DeployVnfApplianceCmd extends DeployVMCmd implements UserCmd { @Parameter(name = ApiConstants.VNF_CONFIGURE_MANAGEMENT, type = CommandType.BOOLEAN, required = false, - description = "True by default, security group or network rules (source nat and firewall rules) will be configured for VNF management interfaces. False otherwise. " + + description = "False by default, security group or network rules (source nat and firewall rules) will be configured for VNF management interfaces. True otherwise. " + "Network rules are configured if management network is an isolated network or shared network with security groups.") private Boolean vnfConfigureManagement; diff --git a/api/src/main/java/org/apache/cloudstack/query/QueryService.java b/api/src/main/java/org/apache/cloudstack/query/QueryService.java index 828f9d5e064..5181ebe2b76 100644 --- a/api/src/main/java/org/apache/cloudstack/query/QueryService.java +++ b/api/src/main/java/org/apache/cloudstack/query/QueryService.java @@ -118,7 +118,7 @@ public interface QueryService { ConfigKey UserVMReadOnlyDetails = new ConfigKey<>(String.class, "user.vm.readonly.details", "Advanced", "dataDiskController, rootDiskController", - "List of read-only VM settings/details as comma separated string", true, ConfigKey.Scope.Global, null, null, null, null, null, ConfigKey.Kind.CSV, null); + "List of read-only VM settings/details as comma separated string", true, ConfigKey.Scope.Global, null, null, null, null, null, ConfigKey.Kind.CSV, null, ""); ConfigKey SortKeyAscending = new ConfigKey<>("Advanced", Boolean.class, "sortkey.algorithm", "true", "Sort algorithm - ascending or descending - to use. For entities that use sort key(template, disk offering, service offering, " + diff --git a/api/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateManager.java b/api/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateManager.java index 6571346ad65..3df59811561 100644 --- a/api/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateManager.java +++ b/api/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateManager.java @@ -29,6 +29,7 @@ import org.apache.cloudstack.api.command.user.template.UpdateVnfTemplateCmd; import org.apache.cloudstack.api.command.user.vm.DeployVnfApplianceCmd; import org.apache.cloudstack.framework.config.ConfigKey; import java.util.List; +import java.util.Map; public interface VnfTemplateManager { @@ -42,11 +43,12 @@ public interface VnfTemplateManager { void updateVnfTemplate(long templateId, UpdateVnfTemplateCmd cmd); - void validateVnfApplianceNics(VirtualMachineTemplate template, List networkIds); + void validateVnfApplianceNics(VirtualMachineTemplate template, List networkIds, Map vmNetworkMap); SecurityGroup createSecurityGroupForVnfAppliance(DataCenter zone, VirtualMachineTemplate template, Account owner, DeployVnfApplianceCmd cmd); void createIsolatedNetworkRulesForVnfAppliance(DataCenter zone, VirtualMachineTemplate template, Account owner, UserVm vm, DeployVnfApplianceCmd cmd) throws InsufficientAddressCapacityException, ResourceAllocationException, ResourceUnavailableException; + } diff --git a/api/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateUtils.java b/api/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateUtils.java index e997a50cec0..16ff2abb564 100644 --- a/api/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateUtils.java +++ b/api/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateUtils.java @@ -16,6 +16,7 @@ // under the License. package org.apache.cloudstack.storage.template; +import com.cloud.agent.api.to.deployasis.OVFNetworkTO; import com.cloud.exception.InvalidParameterValueException; import com.cloud.network.VNF; import com.cloud.storage.Storage; @@ -124,6 +125,9 @@ public class VnfTemplateUtils { public static void validateApiCommandParams(BaseCmd cmd, VirtualMachineTemplate template) { if (cmd instanceof RegisterVnfTemplateCmd) { RegisterVnfTemplateCmd registerCmd = (RegisterVnfTemplateCmd) cmd; + if (registerCmd.isDeployAsIs() && CollectionUtils.isNotEmpty(registerCmd.getVnfNics())) { + throw new InvalidParameterValueException("VNF nics cannot be specified when register a deploy-as-is Template. Please wait until Template settings are read from OVA."); + } validateApiCommandParams(registerCmd.getVnfDetails(), registerCmd.getVnfNics(), registerCmd.getTemplateType()); } else if (cmd instanceof UpdateVnfTemplateCmd) { UpdateVnfTemplateCmd updateCmd = (UpdateVnfTemplateCmd) cmd; @@ -149,4 +153,18 @@ public class VnfTemplateUtils { } } } + + public static void validateDeployAsIsTemplateVnfNics(List ovfNetworks, List vnfNics) { + if (CollectionUtils.isEmpty(vnfNics)) { + return; + } + if (CollectionUtils.isEmpty(ovfNetworks)) { + throw new InvalidParameterValueException("The list of networks read from OVA is empty. Please wait until the template is fully downloaded and processed."); + } + for (VNF.VnfNic vnfNic : vnfNics) { + if (vnfNic.getDeviceId() < ovfNetworks.size() && !vnfNic.isRequired()) { + throw new InvalidParameterValueException(String.format("The VNF nic [device ID: %s ] is required as it is defined in the OVA template.", vnfNic.getDeviceId())); + } + } + } } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/StorageOrchestrationService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/StorageOrchestrationService.java index 8be2015bfef..4af0c806060 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/StorageOrchestrationService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/StorageOrchestrationService.java @@ -22,7 +22,6 @@ import java.util.concurrent.Future; import org.apache.cloudstack.api.response.MigrationResponse; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; -import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService.TemplateApiResult; import org.apache.cloudstack.storage.ImageStoreService.MigrationPolicy; @@ -31,5 +30,5 @@ public interface StorageOrchestrationService { MigrationResponse migrateResources(Long srcImgStoreId, Long destImgStoreId, List templateIdList, List snapshotIdList); - Future orchestrateTemplateCopyToImageStore(TemplateInfo source, DataStore destStore); + Future orchestrateTemplateCopyFromSecondaryStores(long templateId, DataStore destStore); } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotDataFactory.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotDataFactory.java index d318707a29a..32434829c07 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotDataFactory.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotDataFactory.java @@ -30,6 +30,8 @@ public interface SnapshotDataFactory { SnapshotInfo getSnapshot(long snapshotId, long storeId, DataStoreRole role); + SnapshotInfo getSnapshotIncludingRemoved(long snapshotId, long storeId, DataStoreRole role); + SnapshotInfo getSnapshotWithRoleAndZone(long snapshotId, DataStoreRole role, long zoneId); SnapshotInfo getSnapshotOnPrimaryStore(long snapshotId); diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java index a8861d5acc6..269eb4f1c21 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java @@ -80,4 +80,6 @@ public interface TemplateService { List getTemplateDatadisksOnImageStore(TemplateInfo templateInfo, String configurationId); AsyncCallFuture copyTemplateToImageStore(DataObject source, DataStore destStore); -} + + void handleTemplateCopyFromSecondaryStores(long templateId, DataStore destStore); + } diff --git a/engine/components-api/src/main/java/com/cloud/alert/AlertManager.java b/engine/components-api/src/main/java/com/cloud/alert/AlertManager.java index 3d4e6579f7c..7fe19c3ba9f 100644 --- a/engine/components-api/src/main/java/com/cloud/alert/AlertManager.java +++ b/engine/components-api/src/main/java/com/cloud/alert/AlertManager.java @@ -54,5 +54,4 @@ public interface AlertManager extends Manager, AlertService { void recalculateCapacity(); void sendAlert(AlertType alertType, long dataCenterId, Long podId, String subject, String body); - } diff --git a/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java b/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java index 89dc4badcbc..936e8b3448e 100755 --- a/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java +++ b/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java @@ -51,8 +51,8 @@ public interface ResourceManager extends ResourceService, Configurable { ConfigKey KvmSshToAgentEnabled = new ConfigKey<>("Advanced", Boolean.class, "kvm.ssh.to.agent","true", - "Number of retries when preparing a host into Maintenance Mode is faulty before failing", - false); + "True if the management server will restart the agent service via SSH into the KVM hosts after or during maintenance operations", + true); ConfigKey HOST_MAINTENANCE_LOCAL_STRATEGY = new ConfigKey<>(String.class, "host.maintenance.local.storage.strategy", "Advanced","Error", diff --git a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java index de0cb34d63e..4ce1f4a9638 100644 --- a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java +++ b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java @@ -220,8 +220,9 @@ public interface StorageManager extends StorageService { "storage.pool.host.connect.workers", "1", "Number of worker threads to be used to connect hosts to a primary storage", true); - ConfigKey COPY_PUBLIC_TEMPLATES_FROM_OTHER_STORAGES = new ConfigKey<>(Boolean.class, "copy.public.templates.from.other.storages", - "Storage", "true", "Allow SSVMs to try copying public templates from one secondary storage to another instead of downloading them from the source.", + ConfigKey COPY_TEMPLATES_FROM_OTHER_SECONDARY_STORAGES = new ConfigKey<>(Boolean.class, "copy.templates.from.other.secondary.storages", + "Storage", "true", "When enabled, this feature allows templates to be copied from existing Secondary Storage servers (within the same zone or across zones) " + + "while adding a new Secondary Storage. If the copy operation fails, the system falls back to downloading the template from the source URL.", true, ConfigKey.Scope.Zone, null); /** diff --git a/engine/components-api/src/main/java/com/cloud/template/TemplateManager.java b/engine/components-api/src/main/java/com/cloud/template/TemplateManager.java index 7191bd5b1d7..1dde19d7a5d 100644 --- a/engine/components-api/src/main/java/com/cloud/template/TemplateManager.java +++ b/engine/components-api/src/main/java/com/cloud/template/TemplateManager.java @@ -152,6 +152,8 @@ public interface TemplateManager { TemplateType validateTemplateType(BaseCmd cmd, boolean isAdmin, boolean isCrossZones); + DataStore verifyHeuristicRulesForZone(VMTemplateVO template, Long zoneId); + List getTemplateDisksOnImageStore(VirtualMachineTemplate template, DataStoreRole role, String configurationId); static Boolean getValidateUrlIsResolvableBeforeRegisteringTemplateValue() { diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachinePowerStateSyncImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachinePowerStateSyncImpl.java index 7a1a39ec0f0..475ed0f37bd 100644 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachinePowerStateSyncImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachinePowerStateSyncImpl.java @@ -89,7 +89,7 @@ public class VirtualMachinePowerStateSyncImpl implements VirtualMachinePowerStat return; } for (Long vmId : vmIds) { - if (!notUpdated.containsKey(vmId)) { + if (MapUtils.isEmpty(notUpdated) || !notUpdated.containsKey(vmId)) { logger.debug("VM state report is updated. {}, {}, power state: {}", () -> hostCache.get(hostId), () -> vmCache.get(vmId), () -> instancePowerStates.get(vmId)); _messageBus.publish(null, VirtualMachineManager.Topics.VM_POWER_STATE, @@ -158,8 +158,8 @@ public class VirtualMachinePowerStateSyncImpl implements VirtualMachinePowerStat // an update might have occurred that we should not override in case of out of band migration instancePowerStates.put(instance.getId(), VirtualMachine.PowerState.PowerReportMissing); } else { - logger.debug("vm id: {} - time since last state update({} ms) has not passed graceful period yet", - instance.getId(), milliSecondsSinceLastStateUpdate); + logger.debug("vm id: {} - time since last state update({} ms) has not passed graceful period ({} ms) yet", + instance.getId(), milliSecondsSinceLastStateUpdate, milliSecondsGracefulPeriod); } } updateAndPublishVmPowerStates(hostId, instancePowerStates, startTime); diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java index 5aad4892b0f..2281753ca1e 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java @@ -3559,8 +3559,9 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra final HashMap stillFree = new HashMap(); final List networkIds = _networksDao.findNetworksToGarbageCollect(); - final int netGcWait = NumbersUtil.parseInt(_configDao.getValue(NetworkGcWait.key()), 60); - logger.info("NetworkGarbageCollector uses '{}' seconds for GC interval.", netGcWait); + final int netGcWait = NetworkGcWait.value(); + final int netGcInterval = NetworkGcInterval.value(); + logger.info("NetworkGarbageCollector uses '{}' seconds for GC wait and '{}' seconds for GC interval.", netGcWait, netGcInterval); for (final Long networkId : networkIds) { if (!_networkModel.isNetworkReadyForGc(networkId)) { @@ -4882,9 +4883,9 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra } public static final ConfigKey NetworkGcWait = new ConfigKey(Integer.class, "network.gc.wait", "Advanced", "600", - "Time (in seconds) to wait before shutting down a network that's not in used", false, Scope.Global, null); + "Time (in seconds) to wait before shutting down a network that's not in used", true, Scope.Global, null); public static final ConfigKey NetworkGcInterval = new ConfigKey(Integer.class, "network.gc.interval", "Advanced", "600", - "Seconds to wait before checking for networks to shutdown", true, Scope.Global, null); + "Seconds to wait before checking for networks to shutdown", false, Scope.Global, null); @Override public ConfigKey[] getConfigKeys() { diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java index 37a1f8dc196..933b4e0c5ce 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java @@ -36,6 +36,9 @@ import java.util.stream.Collectors; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.template.TemplateManager; import org.apache.cloudstack.api.response.MigrationResponse; import org.apache.cloudstack.engine.orchestration.service.StorageOrchestrationService; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; @@ -45,6 +48,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.SecondaryStorageServic import org.apache.cloudstack.engine.subsystem.api.storage.SecondaryStorageService.DataObjectResult; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService.TemplateApiResult; @@ -103,6 +107,15 @@ public class StorageOrchestrator extends ManagerBase implements StorageOrchestra VolumeDataStoreDao volumeDataStoreDao; @Inject DataMigrationUtility migrationHelper; + @Inject + TemplateManager templateManager; + @Inject + VMTemplateDao templateDao; + @Inject + TemplateDataFactory templateDataFactory; + @Inject + DataCenterDao dcDao; + ConfigKey ImageStoreImbalanceThreshold = new ConfigKey<>("Advanced", Double.class, "image.store.imbalance.threshold", @@ -304,8 +317,9 @@ public class StorageOrchestrator extends ManagerBase implements StorageOrchestra } @Override - public Future orchestrateTemplateCopyToImageStore(TemplateInfo source, DataStore destStore) { - return submit(destStore.getScope().getScopeId(), new CopyTemplateTask(source, destStore)); + public Future orchestrateTemplateCopyFromSecondaryStores(long srcTemplateId, DataStore destStore) { + Long dstZoneId = destStore.getScope().getScopeId(); + return submit(dstZoneId, new CopyTemplateFromSecondaryStorageTask(srcTemplateId, destStore)); } protected Pair migrateCompleted(Long destDatastoreId, DataStore srcDatastore, List files, MigrationPolicy migrationPolicy, int skipped) { @@ -624,13 +638,13 @@ public class StorageOrchestrator extends ManagerBase implements StorageOrchestra } } - private class CopyTemplateTask implements Callable { - private TemplateInfo sourceTmpl; - private DataStore destStore; - private String logid; + private class CopyTemplateFromSecondaryStorageTask implements Callable { + private final long srcTemplateId; + private final DataStore destStore; + private final String logid; - public CopyTemplateTask(TemplateInfo sourceTmpl, DataStore destStore) { - this.sourceTmpl = sourceTmpl; + CopyTemplateFromSecondaryStorageTask(long srcTemplateId, DataStore destStore) { + this.srcTemplateId = srcTemplateId; this.destStore = destStore; this.logid = ThreadContext.get(LOGCONTEXTID); } @@ -639,17 +653,16 @@ public class StorageOrchestrator extends ManagerBase implements StorageOrchestra public TemplateApiResult call() { ThreadContext.put(LOGCONTEXTID, logid); TemplateApiResult result; - AsyncCallFuture future = templateService.copyTemplateToImageStore(sourceTmpl, destStore); + long destZoneId = destStore.getScope().getScopeId(); + TemplateInfo sourceTmpl = templateDataFactory.getTemplate(srcTemplateId, DataStoreRole.Image); try { - result = future.get(); - } catch (ExecutionException | InterruptedException e) { - logger.warn("Exception while copying template [{}] from image store [{}] to image store [{}]: {}", - sourceTmpl.getUniqueName(), sourceTmpl.getDataStore().getName(), destStore.getName(), e.toString()); + templateService.handleTemplateCopyFromSecondaryStores(srcTemplateId, destStore); result = new TemplateApiResult(sourceTmpl); - result.setResult(e.getMessage()); + } finally { + tryCleaningUpExecutor(destZoneId); + ThreadContext.clearAll(); } - tryCleaningUpExecutor(destStore.getScope().getScopeId()); - ThreadContext.clearAll(); + return result; } } diff --git a/engine/schema/src/main/java/com/cloud/network/dao/PhysicalNetworkTrafficTypeDaoImpl.java b/engine/schema/src/main/java/com/cloud/network/dao/PhysicalNetworkTrafficTypeDaoImpl.java index 4811b59d31e..6504bb1f3c8 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/PhysicalNetworkTrafficTypeDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/PhysicalNetworkTrafficTypeDaoImpl.java @@ -137,7 +137,9 @@ public class PhysicalNetworkTrafficTypeDaoImpl extends GenericDaoBase tag = customSearch(sc, null); return tag.size() == 0 ? null : tag.get(0); diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDao.java index 171634fb104..737e4a5a53c 100755 --- a/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDao.java @@ -47,6 +47,8 @@ public interface SnapshotDao extends GenericDao, StateDao listAllByStatus(Snapshot.State... status); + List listAllByStatusIncludingRemoved(Snapshot.State... status); + void updateVolumeIds(long oldVolId, long newVolId); List listByStatusNotIn(long volumeId, Snapshot.State... status); diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDaoImpl.java index f5fc9c47d03..238ae54e07f 100755 --- a/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDaoImpl.java @@ -252,6 +252,13 @@ public class SnapshotDaoImpl extends GenericDaoBase implements return listBy(sc, null); } + @Override + public List listAllByStatusIncludingRemoved(Snapshot.State... status) { + SearchCriteria sc = StatusSearch.create(); + sc.setParameters("status", (Object[])status); + return listIncludingRemovedBy(sc, null); + } + @Override public List listByIds(Object... ids) { SearchCriteria sc = snapshotIdsSearch.create(); diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java index 4936af3caab..83f02719518 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java @@ -48,7 +48,7 @@ public interface VolumeDao extends GenericDao, StateDao findIncludingRemovedByInstanceAndType(long id, Volume.Type vType); - List findByInstanceIdAndPoolId(long instanceId, long poolId); + List findNonDestroyedVolumesByInstanceIdAndPoolId(long instanceId, long poolId); List findByInstanceIdDestroyed(long vmId); @@ -70,11 +70,11 @@ public interface VolumeDao extends GenericDao, StateDao findCreatedByInstance(long id); - List findByPoolId(long poolId); + List findNonDestroyedVolumesByPoolId(long poolId); VolumeVO findByPoolIdName(long poolId, String name); - List findByPoolId(long poolId, Volume.Type volumeType); + List findNonDestroyedVolumesByPoolId(long poolId, Volume.Type volumeType); List findByPoolIdAndState(long poolid, Volume.State state); diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java index 5ef64b04664..a72b4a25845 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java @@ -135,7 +135,7 @@ public class VolumeDaoImpl extends GenericDaoBase implements Vol } @Override - public List findByPoolId(long poolId) { + public List findNonDestroyedVolumesByPoolId(long poolId) { SearchCriteria sc = AllFieldsSearch.create(); sc.setParameters("poolId", poolId); sc.setParameters("notDestroyed", Volume.State.Destroy, Volume.State.Expunged); @@ -144,7 +144,7 @@ public class VolumeDaoImpl extends GenericDaoBase implements Vol } @Override - public List findByInstanceIdAndPoolId(long instanceId, long poolId) { + public List findNonDestroyedVolumesByInstanceIdAndPoolId(long instanceId, long poolId) { SearchCriteria sc = AllFieldsSearch.create(); sc.setParameters("instanceId", instanceId); sc.setParameters("poolId", poolId); @@ -161,7 +161,7 @@ public class VolumeDaoImpl extends GenericDaoBase implements Vol } @Override - public List findByPoolId(long poolId, Volume.Type volumeType) { + public List findNonDestroyedVolumesByPoolId(long poolId, Volume.Type volumeType) { SearchCriteria sc = AllFieldsSearch.create(); sc.setParameters("poolId", poolId); sc.setParameters("notDestroyed", Volume.State.Destroy, Volume.State.Expunged); diff --git a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java index afb7a8d69e6..a8a166fbf27 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java @@ -33,6 +33,7 @@ import java.util.List; import javax.inject.Inject; +import com.cloud.upgrade.dao.Upgrade42020to42030; import com.cloud.utils.FileUtil; import org.apache.cloudstack.utils.CloudStackVersion; import org.apache.commons.lang3.StringUtils; @@ -236,6 +237,7 @@ public class DatabaseUpgradeChecker implements SystemIntegrityChecker { .next("4.19.0.0", new Upgrade41900to41910()) .next("4.19.1.0", new Upgrade41910to42000()) .next("4.20.0.0", new Upgrade42000to42010()) + .next("4.20.2.0", new Upgrade42020to42030()) .build(); } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42020to42030.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42020to42030.java new file mode 100644 index 00000000000..68100e16401 --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42020to42030.java @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.upgrade.dao; + +import java.io.InputStream; +import java.sql.Connection; + +import com.cloud.utils.exception.CloudRuntimeException; + +public class Upgrade42020to42030 extends DbUpgradeAbstractImpl implements DbUpgrade, DbUpgradeSystemVmTemplate { + + @Override + public String[] getUpgradableVersionRange() { + return new String[]{"4.20.2.0", "4.20.3.0"}; + } + + @Override + public String getUpgradedVersion() { + return "4.20.3.0"; + } + + @Override + public boolean supportsRollingUpgrade() { + return false; + } + + @Override + public InputStream[] getPrepareScripts() { + final String scriptFile = "META-INF/db/schema-42020to42030.sql"; + final InputStream script = Thread.currentThread().getContextClassLoader().getResourceAsStream(scriptFile); + if (script == null) { + throw new CloudRuntimeException("Unable to find " + scriptFile); + } + + return new InputStream[] {script}; + } + + @Override + public void performDataMigration(Connection conn) { + } + + @Override + public InputStream[] getCleanupScripts() { + return null; + } + + @Override + public void updateSystemVmTemplates(Connection conn) { + } +} diff --git a/engine/schema/src/main/java/com/cloud/usage/dao/UsageDao.java b/engine/schema/src/main/java/com/cloud/usage/dao/UsageDao.java index ea490e60f9e..b9d8d9c0536 100644 --- a/engine/schema/src/main/java/com/cloud/usage/dao/UsageDao.java +++ b/engine/schema/src/main/java/com/cloud/usage/dao/UsageDao.java @@ -62,7 +62,7 @@ public interface UsageDao extends GenericDao { void saveUsageRecords(List usageRecords); - void removeOldUsageRecords(int days); + void expungeAllOlderThan(int days, long limitPerQuery); UsageVO persistUsage(final UsageVO usage); diff --git a/engine/schema/src/main/java/com/cloud/usage/dao/UsageDaoImpl.java b/engine/schema/src/main/java/com/cloud/usage/dao/UsageDaoImpl.java index d863aacfd44..2d99c78fad1 100644 --- a/engine/schema/src/main/java/com/cloud/usage/dao/UsageDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/usage/dao/UsageDaoImpl.java @@ -26,15 +26,16 @@ import com.cloud.utils.Pair; import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.QueryBuilder; +import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.Transaction; import com.cloud.utils.db.TransactionCallback; -import com.cloud.utils.db.TransactionCallbackNoReturn; import com.cloud.utils.db.TransactionLegacy; import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.acl.RoleType; +import org.apache.commons.lang3.time.DateUtils; import org.springframework.stereotype.Component; import java.sql.PreparedStatement; @@ -51,8 +52,7 @@ import java.util.TimeZone; public class UsageDaoImpl extends GenericDaoBase implements UsageDao { private static final String DELETE_ALL = "DELETE FROM cloud_usage"; private static final String DELETE_ALL_BY_ACCOUNTID = "DELETE FROM cloud_usage WHERE account_id = ?"; - private static final String DELETE_ALL_BY_INTERVAL = "DELETE FROM cloud_usage WHERE end_date < DATE_SUB(CURRENT_DATE(), INTERVAL ? DAY)"; - private static final String INSERT_ACCOUNT = "INSERT INTO cloud_usage.account (id, account_name, type, role_id, domain_id, removed, cleanup_needed) VALUES (?,?,?,?,?,?,?)"; + private static final String INSERT_ACCOUNT = "INSERT INTO cloud_usage.account (id, account_name, uuid, type, role_id, domain_id, removed, cleanup_needed) VALUES (?,?,?,?,?,?,?,?)"; private static final String INSERT_USER_STATS = "INSERT INTO cloud_usage.user_statistics (id, data_center_id, account_id, public_ip_address, device_id, device_type, network_id, net_bytes_received," + " net_bytes_sent, current_bytes_received, current_bytes_sent, agg_bytes_received, agg_bytes_sent) VALUES (?,?,?,?,?,?,?,?,?,?, ?, ?, ?)"; @@ -88,8 +88,12 @@ public class UsageDaoImpl extends GenericDaoBase implements Usage private static final String UPDATE_BUCKET_STATS = "UPDATE cloud_usage.bucket_statistics SET size=? WHERE id=?"; + protected SearchBuilder endDateLessThanSearch; public UsageDaoImpl() { + endDateLessThanSearch = createSearchBuilder(); + endDateLessThanSearch.and("endDate", endDateLessThanSearch.entity().getEndDate(), SearchCriteria.Op.LT); + endDateLessThanSearch.done(); } @Override @@ -129,25 +133,26 @@ public class UsageDaoImpl extends GenericDaoBase implements Usage for (AccountVO acct : accounts) { pstmt.setLong(1, acct.getId()); pstmt.setString(2, acct.getAccountName()); - pstmt.setInt(3, acct.getType().ordinal()); + pstmt.setString(3, acct.getUuid()); + pstmt.setInt(4, acct.getType().ordinal()); //prevent autoboxing NPE by defaulting to User role if(acct.getRoleId() == null){ - pstmt.setLong(4, RoleType.User.getId()); + pstmt.setLong(5, RoleType.User.getId()); }else{ - pstmt.setLong(4, acct.getRoleId()); + pstmt.setLong(5, acct.getRoleId()); } - pstmt.setLong(5, acct.getDomainId()); + pstmt.setLong(6, acct.getDomainId()); Date removed = acct.getRemoved(); if (removed == null) { - pstmt.setString(6, null); + pstmt.setString(7, null); } else { - pstmt.setString(6, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), acct.getRemoved())); + pstmt.setString(7, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), acct.getRemoved())); } - pstmt.setBoolean(7, acct.getNeedsCleanup()); + pstmt.setBoolean(8, acct.getNeedsCleanup()); pstmt.addBatch(); } @@ -538,21 +543,20 @@ public class UsageDaoImpl extends GenericDaoBase implements Usage } @Override - public void removeOldUsageRecords(int days) { - Transaction.execute(TransactionLegacy.USAGE_DB, new TransactionCallbackNoReturn() { - @Override - public void doInTransactionWithoutResult(TransactionStatus status) { - TransactionLegacy txn = TransactionLegacy.currentTxn(); - PreparedStatement pstmt = null; - try { - pstmt = txn.prepareAutoCloseStatement(DELETE_ALL_BY_INTERVAL); - pstmt.setLong(1, days); - pstmt.executeUpdate(); - } catch (Exception ex) { - logger.error("error removing old cloud_usage records for interval: " + days); - } - } - }); + public void expungeAllOlderThan(int days, long limitPerQuery) { + SearchCriteria sc = endDateLessThanSearch.create(); + + Date limit = DateUtils.addDays(new Date(), -days); + sc.setParameters("endDate", limit); + + TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.USAGE_DB); + try { + logger.debug("Removing all cloud_usage records older than [{}].", limit); + int totalExpunged = batchExpunge(sc, limitPerQuery); + logger.info("Removed a total of [{}] cloud_usage records older than [{}].", totalExpunged, limit); + } finally { + txn.close(); + } } public UsageVO persistUsage(final UsageVO usage) { diff --git a/engine/schema/src/main/java/com/cloud/usage/dao/UsageJobDao.java b/engine/schema/src/main/java/com/cloud/usage/dao/UsageJobDao.java index d4038d4ceeb..b22ce69d94e 100644 --- a/engine/schema/src/main/java/com/cloud/usage/dao/UsageJobDao.java +++ b/engine/schema/src/main/java/com/cloud/usage/dao/UsageJobDao.java @@ -28,6 +28,8 @@ public interface UsageJobDao extends GenericDao { UsageJobVO getLastJob(); + UsageJobVO getNextRecurringJob(); + UsageJobVO getNextImmediateJob(); long getLastJobSuccessDateMillis(); diff --git a/engine/schema/src/main/java/com/cloud/usage/dao/UsageJobDaoImpl.java b/engine/schema/src/main/java/com/cloud/usage/dao/UsageJobDaoImpl.java index 44a7d1a8b72..6f340501cf1 100644 --- a/engine/schema/src/main/java/com/cloud/usage/dao/UsageJobDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/usage/dao/UsageJobDaoImpl.java @@ -156,7 +156,8 @@ public class UsageJobDaoImpl extends GenericDaoBase implements return jobs.get(0); } - private UsageJobVO getNextRecurringJob() { + @Override + public UsageJobVO getNextRecurringJob() { Filter filter = new Filter(UsageJobVO.class, "id", false, Long.valueOf(0), Long.valueOf(1)); SearchCriteria sc = createSearchCriteria(); sc.addAnd("endMillis", SearchCriteria.Op.EQ, Long.valueOf(0)); diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java index 4cd29b465ee..96df4928773 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java @@ -56,6 +56,8 @@ StateDao findBySnapshotId(long snapshotId); + List findBySnapshotIdWithNonDestroyedState(long snapshotId); + void duplicateCacheRecordsOnRegionStore(long storeId); // delete the snapshot entry on primary data store to make sure that next snapshot will be full snapshot @@ -108,4 +110,18 @@ StateDao snapshotIds, Long batchSize); + + /** + * Returns the total physical size, in bytes, of all snapshots stored on primary + * storage for the specified account that have not yet been backed up to + * secondary storage. + * + *

If no such snapshots are found, this method returns {@code 0}.

+ * + * @param accountId the ID of the account whose snapshots on primary storage + * should be considered + * @return the total physical size in bytes of matching snapshots on primary + * storage, or {@code 0} if none are found + */ + long getSnapshotsPhysicalSizeOnPrimaryStorageByAccountId(long accountId); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java index 5bf67eb3881..c68316dd1fe 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java @@ -78,6 +78,15 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase params) throws ConfigurationException { super.configure(name, params); @@ -118,7 +127,6 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase findBySnapshotId(long snapshotId) { + SearchCriteria sc = searchFilteringStoreIdEqStateEqStoreRoleEqIdEqUpdateCountEqSnapshotIdEqVolumeIdEq.create(); + sc.setParameters(SNAPSHOT_ID, snapshotId); + return listBy(sc); + } + + @Override + public List findBySnapshotIdWithNonDestroyedState(long snapshotId) { SearchCriteria sc = idStateNeqSearch.create(); sc.setParameters(SNAPSHOT_ID, snapshotId); sc.setParameters(STATE, State.Destroyed); @@ -571,4 +586,23 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase { int expungeJobsForSchedules(List scheduleId, Date dateAfter); int expungeJobsBefore(Date currentTimestamp); + + VMScheduledJobVO findByScheduleAndTimestamp(long scheduleId, Date scheduledTimestamp); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/vm/schedule/dao/VMScheduledJobDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/vm/schedule/dao/VMScheduledJobDaoImpl.java index 50a2b12fd77..2f08a41b92e 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/vm/schedule/dao/VMScheduledJobDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/vm/schedule/dao/VMScheduledJobDaoImpl.java @@ -39,6 +39,8 @@ public class VMScheduledJobDaoImpl extends GenericDaoBase expungeJobForScheduleSearch; + private final SearchBuilder scheduleAndTimestampSearch; + static final String SCHEDULED_TIMESTAMP = "scheduled_timestamp"; static final String VM_SCHEDULE_ID = "vm_schedule_id"; @@ -58,6 +60,11 @@ public class VMScheduledJobDaoImpl extends GenericDaoBase sc = scheduleAndTimestampSearch.create(); + sc.setParameters(VM_SCHEDULE_ID, scheduleId); + sc.setParameters(SCHEDULED_TIMESTAMP, scheduledTimestamp); + return findOneBy(sc); + } } diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42020to42030.sql b/engine/schema/src/main/resources/META-INF/db/schema-42020to42030.sql new file mode 100644 index 00000000000..5eec97278ba --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/schema-42020to42030.sql @@ -0,0 +1,28 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +--; +-- Schema upgrade from 4.20.2.0 to 4.20.3.0 +--; + +ALTER TABLE `cloud`.`template_store_ref` MODIFY COLUMN `download_url` varchar(2048); + +UPDATE `cloud`.`alert` SET type = 33 WHERE name = 'ALERT.VR.PUBLIC.IFACE.MTU'; +UPDATE `cloud`.`alert` SET type = 34 WHERE name = 'ALERT.VR.PRIVATE.IFACE.MTU'; + +-- Update configuration 'kvm.ssh.to.agent' description and is_dynamic fields +UPDATE `cloud`.`configuration` SET description = 'True if the management server will restart the agent service via SSH into the KVM hosts after or during maintenance operations', is_dynamic = 1 WHERE name = 'kvm.ssh.to.agent'; diff --git a/engine/storage/configdrive/src/main/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilder.java b/engine/storage/configdrive/src/main/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilder.java index 0b81a25b1cd..15febbe972c 100644 --- a/engine/storage/configdrive/src/main/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilder.java +++ b/engine/storage/configdrive/src/main/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilder.java @@ -231,9 +231,9 @@ public class ConfigDriveBuilder { throw new CloudRuntimeException("Cannot create ISO for config drive using any know tool. Known paths [/usr/bin/genisoimage, /usr/bin/mkisofs, /usr/local/bin/mkisofs]"); } if (!isoCreator.canExecute()) { - throw new CloudRuntimeException("Cannot create ISO for config drive using: " + isoCreator.getCanonicalPath()); + throw new CloudRuntimeException("Cannot create ISO for config drive using: " + isoCreator.getAbsolutePath()); } - return isoCreator.getCanonicalPath(); + return isoCreator.getAbsolutePath(); } /** diff --git a/engine/storage/configdrive/src/test/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilderTest.java b/engine/storage/configdrive/src/test/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilderTest.java index c04ff0a1601..03ceac84399 100644 --- a/engine/storage/configdrive/src/test/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilderTest.java +++ b/engine/storage/configdrive/src/test/java/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilderTest.java @@ -435,7 +435,7 @@ public class ConfigDriveBuilderTest { Mockito.verify(genIsoFileMock, Mockito.times(2)).exists(); Mockito.verify(genIsoFileMock).canExecute(); - Mockito.verify(genIsoFileMock).getCanonicalPath(); + Mockito.verify(genIsoFileMock).getAbsolutePath(); } } @@ -475,11 +475,11 @@ public class ConfigDriveBuilderTest { Mockito.verify(genIsoFileMock, Mockito.times(1)).exists(); Mockito.verify(genIsoFileMock, Mockito.times(0)).canExecute(); - Mockito.verify(genIsoFileMock, Mockito.times(0)).getCanonicalPath(); + Mockito.verify(genIsoFileMock, Mockito.times(0)).getAbsolutePath(); Mockito.verify(mkIsoProgramInLinuxFileMock, Mockito.times(2)).exists(); Mockito.verify(mkIsoProgramInLinuxFileMock, Mockito.times(1)).canExecute(); - Mockito.verify(mkIsoProgramInLinuxFileMock, Mockito.times(1)).getCanonicalPath(); + Mockito.verify(mkIsoProgramInLinuxFileMock, Mockito.times(1)).getAbsolutePath(); } } @@ -509,15 +509,15 @@ public class ConfigDriveBuilderTest { Mockito.verify(genIsoFileMock, Mockito.times(1)).exists(); Mockito.verify(genIsoFileMock, Mockito.times(0)).canExecute(); - Mockito.verify(genIsoFileMock, Mockito.times(0)).getCanonicalPath(); + Mockito.verify(genIsoFileMock, Mockito.times(0)).getAbsolutePath(); Mockito.verify(mkIsoProgramInLinuxFileMock, Mockito.times(1)).exists(); Mockito.verify(mkIsoProgramInLinuxFileMock, Mockito.times(0)).canExecute(); - Mockito.verify(mkIsoProgramInLinuxFileMock, Mockito.times(0)).getCanonicalPath(); + Mockito.verify(mkIsoProgramInLinuxFileMock, Mockito.times(0)).getAbsolutePath(); Mockito.verify(mkIsoProgramInMacOsFileMock, Mockito.times(1)).exists(); Mockito.verify(mkIsoProgramInMacOsFileMock, Mockito.times(1)).canExecute(); - Mockito.verify(mkIsoProgramInMacOsFileMock, Mockito.times(1)).getCanonicalPath(); + Mockito.verify(mkIsoProgramInMacOsFileMock, Mockito.times(1)).getAbsolutePath(); } } diff --git a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java index b59ee2c6166..4cb09fb81f7 100644 --- a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java +++ b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java @@ -45,10 +45,12 @@ import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; import org.apache.cloudstack.framework.async.AsyncCompletionCallback; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.secstorage.heuristics.HeuristicType; import org.apache.cloudstack.storage.RemoteHostEndPoint; import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao; import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO; +import org.apache.cloudstack.storage.heuristics.HeuristicRuleHelper; import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; import org.apache.logging.log4j.Logger; @@ -104,6 +106,9 @@ public class AncientDataMotionStrategy implements DataMotionStrategy { @Inject SnapshotDao snapshotDao; + @Inject + HeuristicRuleHelper heuristicRuleHelper; + @Override public StrategyPriority canHandle(DataObject srcData, DataObject destData) { return StrategyPriority.DEFAULT; @@ -374,7 +379,13 @@ public class AncientDataMotionStrategy implements DataMotionStrategy { } // need to find a nfs or cifs image store, assuming that can't copy volume // directly to s3 - ImageStoreEntity imageStore = (ImageStoreEntity)dataStoreMgr.getImageStoreWithFreeCapacity(destScope.getScopeId()); + Long zoneId = destScope.getScopeId(); + ImageStoreEntity imageStore = (ImageStoreEntity) heuristicRuleHelper.getImageStoreIfThereIsHeuristicRule(zoneId, HeuristicType.VOLUME, destData); + if (imageStore == null) { + logger.debug("Secondary storage selector did not direct volume migration to a specific secondary storage; using secondary storage with the most free capacity."); + imageStore = (ImageStoreEntity) dataStoreMgr.getImageStoreWithFreeCapacity(zoneId); + } + if (imageStore == null || !imageStore.getProtocol().equalsIgnoreCase("nfs") && !imageStore.getProtocol().equalsIgnoreCase("cifs")) { String errMsg = "can't find a nfs (or cifs) image store to satisfy the need for a staging store"; Answer answer = new Answer(null, false, errMsg); diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/SecondaryStorageServiceImpl.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/SecondaryStorageServiceImpl.java index 641a2a40dcd..f739fecf9bf 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/SecondaryStorageServiceImpl.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/SecondaryStorageServiceImpl.java @@ -280,7 +280,7 @@ public class SecondaryStorageServiceImpl implements SecondaryStorageService { private void updateDataObject(DataObject srcData, DataObject destData) { if (destData instanceof SnapshotInfo) { SnapshotDataStoreVO snapshotStore = snapshotStoreDao.findBySourceSnapshot(srcData.getId(), DataStoreRole.Image); - SnapshotDataStoreVO destSnapshotStore = snapshotStoreDao.findByStoreSnapshot(DataStoreRole.Image, srcData.getDataStore().getId(), srcData.getId()); + SnapshotDataStoreVO destSnapshotStore = snapshotStoreDao.findByStoreSnapshot(DataStoreRole.Image, destData.getDataStore().getId(), destData.getId()); if (snapshotStore != null && destSnapshotStore != null) { destSnapshotStore.setPhysicalSize(snapshotStore.getPhysicalSize()); destSnapshotStore.setCreated(snapshotStore.getCreated()); diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java index c6430bcf9f9..3e1504beb3a 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java @@ -296,6 +296,9 @@ public class TemplateDataFactoryImpl implements TemplateDataFactory { @Override public boolean isTemplateMarkedForDirectDownload(long templateId) { VMTemplateVO templateVO = imageDataDao.findById(templateId); + if (templateVO == null) { + throw new CloudRuntimeException(String.format("Template not found with ID: %s", templateId)); + } return templateVO.isDirectDownload(); } } diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java index 1bb954da410..5fc9bbac352 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java @@ -31,6 +31,8 @@ import java.util.concurrent.ExecutionException; import javax.inject.Inject; +import com.cloud.exception.StorageUnavailableException; +import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.orchestration.service.StorageOrchestrationService; import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult; @@ -67,9 +69,11 @@ import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; import org.apache.cloudstack.storage.image.store.TemplateObject; import org.apache.cloudstack.storage.to.TemplateObjectTO; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.ThreadContext; import org.springframework.stereotype.Component; import com.cloud.agent.api.Answer; @@ -290,21 +294,41 @@ public class TemplateServiceImpl implements TemplateService { } } - protected boolean isSkipTemplateStoreDownload(VMTemplateVO template, Long zoneId) { + protected boolean shouldDownloadTemplateToStore(VMTemplateVO template, DataStore store) { + Long zoneId = store.getScope().getScopeId(); + DataStore directedStore = _tmpltMgr.verifyHeuristicRulesForZone(template, zoneId); + if (directedStore != null && store.getId() != directedStore.getId()) { + logger.info("Template [{}] will not be download to image store [{}], as a heuristic rule is directing it to another store.", + template.getUniqueName(), store.getName()); + return false; + } + if (template.isPublicTemplate()) { - return false; + logger.debug("Download of template [{}] to image store [{}] cannot be skipped, as it is public.", template.getUniqueName(), + store.getName()); + return true; } + if (template.isFeatured()) { - return false; + logger.debug("Download of template [{}] to image store [{}] cannot be skipped, as it is featured.", template.getUniqueName(), + store.getName()); + return true; } + if (TemplateType.SYSTEM.equals(template.getTemplateType())) { - return false; + logger.debug("Download of template [{}] to image store [{}] cannot be skipped, as it is a system VM template.", + template.getUniqueName(),store.getName()); + return true; } + if (zoneId != null && _vmTemplateStoreDao.findByTemplateZone(template.getId(), zoneId, DataStoreRole.Image) == null) { - logger.debug("Template {} is not present on any image store for the zone ID: {}, its download cannot be skipped", template, zoneId); - return false; + logger.debug("Download of template [{}] to image store [{}] cannot be skipped, as it is not present on any image store of zone [{}].", + template.getUniqueName(), store.getName(), zoneId); + return true; } - return true; + + logger.info("Skipping download of template [{}] to image store [{}].", template.getUniqueName(), store.getName()); + return false; } @Override @@ -531,8 +555,7 @@ public class TemplateServiceImpl implements TemplateService { // download. for (VMTemplateVO tmplt : toBeDownloaded) { // if this is private template, skip sync to a new image store - if (isSkipTemplateStoreDownload(tmplt, zoneId)) { - logger.info("Skip sync downloading private template {} to a new image store", tmplt); + if (!shouldDownloadTemplateToStore(tmplt, store)) { continue; } @@ -548,10 +571,7 @@ public class TemplateServiceImpl implements TemplateService { } if (availHypers.contains(tmplt.getHypervisorType())) { - boolean copied = isCopyFromOtherStoragesEnabled(zoneId) && tryCopyingTemplateToImageStore(tmplt, store); - if (!copied) { - tryDownloadingTemplateToImageStore(tmplt, store); - } + storageOrchestrator.orchestrateTemplateCopyFromSecondaryStores(tmplt.getId(), store); } else { logger.info("Skip downloading template {} since current data center does not have hypervisor {}", tmplt, tmplt.getHypervisorType()); } @@ -598,6 +618,16 @@ public class TemplateServiceImpl implements TemplateService { } + @Override + public void handleTemplateCopyFromSecondaryStores(long templateId, DataStore destStore) { + VMTemplateVO template = _templateDao.findById(templateId); + long zoneId = destStore.getScope().getScopeId(); + boolean copied = imageStoreDetailsUtil.isCopyTemplatesFromOtherStoragesEnabled(destStore.getId(), zoneId) && tryCopyingTemplateToImageStore(template, destStore); + if (!copied) { + tryDownloadingTemplateToImageStore(template, destStore); + } + } + protected void tryDownloadingTemplateToImageStore(VMTemplateVO tmplt, DataStore destStore) { if (tmplt.getUrl() == null) { logger.info("Not downloading template [{}] to image store [{}], as it has no URL.", tmplt.getUniqueName(), @@ -615,28 +645,134 @@ public class TemplateServiceImpl implements TemplateService { } protected boolean tryCopyingTemplateToImageStore(VMTemplateVO tmplt, DataStore destStore) { - Long zoneId = destStore.getScope().getScopeId(); - List storesInZone = _storeMgr.getImageStoresByZoneIds(zoneId); - for (DataStore sourceStore : storesInZone) { - Map existingTemplatesInSourceStore = listTemplate(sourceStore); - if (existingTemplatesInSourceStore == null || !existingTemplatesInSourceStore.containsKey(tmplt.getUniqueName())) { - logger.debug("Template [{}] does not exist on image store [{}]; searching on another one.", - tmplt.getUniqueName(), sourceStore.getName()); - continue; - } - TemplateObject sourceTmpl = (TemplateObject) _templateFactory.getTemplate(tmplt.getId(), sourceStore); - if (sourceTmpl.getInstallPath() == null) { - logger.warn("Can not copy template [{}] from image store [{}], as it returned a null install path.", tmplt.getUniqueName(), - sourceStore.getName()); - continue; - } - storageOrchestrator.orchestrateTemplateCopyToImageStore(sourceTmpl, destStore); + if (searchAndCopyWithinZone(tmplt, destStore)) { return true; } - logger.debug("Can't copy template [{}] from another image store.", tmplt.getUniqueName()); + + Long destZoneId = destStore.getScope().getScopeId(); + logger.debug("Template [{}] not found in any image store of zone [{}]. Checking other zones.", + tmplt.getUniqueName(), destZoneId); + + return searchAndCopyAcrossZones(tmplt, destStore, destZoneId); + } + + private boolean searchAndCopyAcrossZones(VMTemplateVO tmplt, DataStore destStore, Long destZoneId) { + List allZoneIds = _dcDao.listAllIds(); + for (Long otherZoneId : allZoneIds) { + if (otherZoneId.equals(destZoneId)) { + continue; + } + + List storesInOtherZone = _storeMgr.getImageStoresByZoneIds(otherZoneId); + logger.debug("Checking zone [{}] for template [{}]...", otherZoneId, tmplt.getUniqueName()); + + if (CollectionUtils.isEmpty(storesInOtherZone)) { + logger.debug("Zone [{}] has no image stores. Skipping.", otherZoneId); + continue; + } + + TemplateObject sourceTmpl = findUsableTemplate(tmplt, storesInOtherZone); + if (sourceTmpl == null) { + logger.debug("Template [{}] not found with a valid install path in any image store of zone [{}].", + tmplt.getUniqueName(), otherZoneId); + continue; + } + + logger.info("Template [{}] found in zone [{}]. Initiating cross-zone copy to zone [{}].", + tmplt.getUniqueName(), otherZoneId, destZoneId); + + return copyTemplateAcrossZones(destStore, sourceTmpl); + } + + logger.debug("Template [{}] was not found in any zone. Cannot perform zone-to-zone copy.", tmplt.getUniqueName()); return false; } + protected TemplateObject findUsableTemplate(VMTemplateVO tmplt, List imageStores) { + for (DataStore store : imageStores) { + + Map templates = listTemplate(store); + if (templates == null || !templates.containsKey(tmplt.getUniqueName())) { + continue; + } + + TemplateObject tmpl = (TemplateObject) _templateFactory.getTemplate(tmplt.getId(), store); + if (tmpl.getInstallPath() == null) { + logger.debug("Template [{}] found in image store [{}] but install path is null. Skipping.", + tmplt.getUniqueName(), store.getName()); + continue; + } + return tmpl; + } + return null; + } + + private boolean searchAndCopyWithinZone(VMTemplateVO tmplt, DataStore destStore) { + Long destZoneId = destStore.getScope().getScopeId(); + List storesInSameZone = _storeMgr.getImageStoresByZoneIds(destZoneId); + + TemplateObject sourceTmpl = findUsableTemplate(tmplt, storesInSameZone); + if (sourceTmpl == null) { + return false; + } + + TemplateApiResult result; + AsyncCallFuture future = copyTemplateToImageStore(sourceTmpl, destStore); + try { + result = future.get(); + } catch (ExecutionException | InterruptedException e) { + logger.warn("Exception while copying template [{}] from image store [{}] to image store [{}]: {}", + sourceTmpl.getUniqueName(), sourceTmpl.getDataStore().getName(), destStore.getName(), e.toString()); + result = new TemplateApiResult(sourceTmpl); + result.setResult(e.getMessage()); + } + return result.isSuccess(); + } + + private boolean copyTemplateAcrossZones(DataStore destStore, TemplateObject sourceTmpl) { + Long dstZoneId = destStore.getScope().getScopeId(); + DataCenterVO dstZone = _dcDao.findById(dstZoneId); + + if (dstZone == null) { + logger.warn("Destination zone [{}] not found for template [{}].", dstZoneId, sourceTmpl.getUniqueName()); + return false; + } + + TemplateApiResult result; + try { + VMTemplateVO template = _templateDao.findById(sourceTmpl.getId()); + try { + DataStore sourceStore = sourceTmpl.getDataStore(); + long userId = CallContext.current().getCallingUserId(); + boolean success = _tmpltMgr.copy(userId, template, sourceStore, dstZone); + + result = new TemplateApiResult(sourceTmpl); + if (!success) { + result.setResult("Cross-zone template copy failed"); + } + } catch (StorageUnavailableException | ResourceAllocationException e) { + logger.error("Exception while copying template [{}] from zone [{}] to zone [{}]", + template, + sourceTmpl.getDataStore().getScope().getScopeId(), + dstZone.getId(), + e); + result = new TemplateApiResult(sourceTmpl); + result.setResult(e.getMessage()); + } finally { + ThreadContext.clearAll(); + } + } catch (Exception e) { + logger.error("Failed to copy template [{}] from zone [{}] to zone [{}].", + sourceTmpl.getUniqueName(), + sourceTmpl.getDataStore().getScope().getScopeId(), + dstZoneId, + e); + return false; + } + + return result.isSuccess(); + } + @Override public AsyncCallFuture copyTemplateToImageStore(DataObject source, DataStore destStore) { TemplateObject sourceTmpl = (TemplateObject) source; @@ -680,10 +816,6 @@ public class TemplateServiceImpl implements TemplateService { return null; } - protected boolean isCopyFromOtherStoragesEnabled(Long zoneId) { - return StorageManager.COPY_PUBLIC_TEMPLATES_FROM_OTHER_STORAGES.valueIn(zoneId); - } - protected void publishTemplateCreation(TemplateInfo tmplt) { VMTemplateVO tmpltVo = _templateDao.findById(tmplt.getId()); @@ -1318,9 +1450,10 @@ public class TemplateServiceImpl implements TemplateService { if (_vmTemplateStoreDao.isTemplateMarkedForDirectDownload(tmplt.getId())) { continue; } - tmpltStore = - new TemplateDataStoreVO(storeId, tmplt.getId(), new Date(), 100, Status.DOWNLOADED, null, null, null, - TemplateConstants.DEFAULT_SYSTEM_VM_TEMPLATE_PATH + tmplt.getId() + '/', tmplt.getUrl()); + String templateDirectoryPath = TemplateConstants.DEFAULT_TMPLT_ROOT_DIR + File.separator + TemplateConstants.DEFAULT_TMPLT_FIRST_LEVEL_DIR; + String installPath = templateDirectoryPath + tmplt.getAccountId() + File.separator + tmplt.getId() + File.separator; + tmpltStore = new TemplateDataStoreVO(storeId, tmplt.getId(), new Date(), 100, Status.DOWNLOADED, + null, null, null, installPath, tmplt.getUrl()); tmpltStore.setSize(0L); tmpltStore.setPhysicalSize(0); // no size information for // pre-seeded system vm templates diff --git a/engine/storage/image/src/test/java/org/apache/cloudstack/storage/image/TemplateServiceImplTest.java b/engine/storage/image/src/test/java/org/apache/cloudstack/storage/image/TemplateServiceImplTest.java index 276581e2e48..e9eac045869 100644 --- a/engine/storage/image/src/test/java/org/apache/cloudstack/storage/image/TemplateServiceImplTest.java +++ b/engine/storage/image/src/test/java/org/apache/cloudstack/storage/image/TemplateServiceImplTest.java @@ -18,12 +18,20 @@ */ package org.apache.cloudstack.storage.image; +import com.cloud.dc.DataCenterVO; +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.StorageUnavailableException; +import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.template.TemplateProp; +import com.cloud.template.TemplateManager; +import com.cloud.user.Account; +import com.cloud.user.User; +import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.orchestration.service.StorageOrchestrationService; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.Scope; -import org.apache.cloudstack.framework.async.AsyncCallFuture; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import org.apache.cloudstack.storage.image.store.TemplateObject; @@ -45,6 +53,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import static org.mockito.Mockito.mock; + @RunWith(MockitoJUnitRunner.class) public class TemplateServiceImplTest { @@ -70,6 +80,9 @@ public class TemplateServiceImplTest { @Mock TemplateObject templateInfoMock; + @Mock + DataStore dataStoreMock; + @Mock DataStore sourceStoreMock; @@ -82,6 +95,15 @@ public class TemplateServiceImplTest { @Mock StorageOrchestrationService storageOrchestrator; + @Mock + TemplateManager templateManagerMock; + + @Mock + VMTemplateDao templateDao; + + @Mock + DataCenterDao _dcDao; + Map templatesInSourceStore = new HashMap<>(); @Before @@ -94,47 +116,46 @@ public class TemplateServiceImplTest { Mockito.doReturn(List.of(sourceStoreMock, destStoreMock)).when(dataStoreManagerMock).getImageStoresByZoneIds(zoneId); Mockito.doReturn(templatesInSourceStore).when(templateService).listTemplate(sourceStoreMock); Mockito.doReturn(null).when(templateService).listTemplate(destStoreMock); - Mockito.doReturn("install-path").when(templateInfoMock).getInstallPath(); Mockito.doReturn(templateInfoMock).when(templateDataFactoryMock).getTemplate(2L, sourceStoreMock); + Mockito.doReturn(3L).when(dataStoreMock).getId(); + Mockito.doReturn(zoneScopeMock).when(dataStoreMock).getScope(); } @Test - public void testIsSkipTemplateStoreDownloadPublicTemplate() { - VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class); - Mockito.when(templateVO.isPublicTemplate()).thenReturn(true); - Assert.assertFalse(templateService.isSkipTemplateStoreDownload(templateVO, 1L)); + public void shouldDownloadTemplateToStoreTestSkipsTemplateDirectedToAnotherStorage() { + DataStore destinedStore = Mockito.mock(DataStore.class); + Mockito.doReturn(dataStoreMock.getId() + 1L).when(destinedStore).getId(); + Mockito.when(templateManagerMock.verifyHeuristicRulesForZone(tmpltMock, zoneScopeMock.getScopeId())).thenReturn(destinedStore); + Assert.assertFalse(templateService.shouldDownloadTemplateToStore(tmpltMock, dataStoreMock)); } @Test - public void testIsSkipTemplateStoreDownloadFeaturedTemplate() { - VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class); - Mockito.when(templateVO.isFeatured()).thenReturn(true); - Assert.assertFalse(templateService.isSkipTemplateStoreDownload(templateVO, 1L)); + public void shouldDownloadTemplateToStoreTestDownloadsPublicTemplate() { + Mockito.when(tmpltMock.isPublicTemplate()).thenReturn(true); + Assert.assertTrue(templateService.shouldDownloadTemplateToStore(tmpltMock, dataStoreMock)); } @Test - public void testIsSkipTemplateStoreDownloadSystemTemplate() { - VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class); - Mockito.when(templateVO.getTemplateType()).thenReturn(Storage.TemplateType.SYSTEM); - Assert.assertFalse(templateService.isSkipTemplateStoreDownload(templateVO, 1L)); + public void shouldDownloadTemplateToStoreTestDownloadsFeaturedTemplate() { + Mockito.when(tmpltMock.isFeatured()).thenReturn(true); + Assert.assertTrue(templateService.shouldDownloadTemplateToStore(tmpltMock, dataStoreMock)); } @Test - public void testIsSkipTemplateStoreDownloadPrivateNoRefTemplate() { - VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class); - long id = 1L; - Mockito.when(templateVO.getId()).thenReturn(id); - Mockito.when(templateDataStoreDao.findByTemplateZone(id, id, DataStoreRole.Image)).thenReturn(null); - Assert.assertFalse(templateService.isSkipTemplateStoreDownload(templateVO, id)); + public void shouldDownloadTemplateToStoreTestDownloadsSystemTemplate() { + Mockito.when(tmpltMock.getTemplateType()).thenReturn(Storage.TemplateType.SYSTEM); + Assert.assertTrue(templateService.shouldDownloadTemplateToStore(tmpltMock, dataStoreMock)); } @Test - public void testIsSkipTemplateStoreDownloadPrivateExistingTemplate() { - VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class); - long id = 1L; - Mockito.when(templateVO.getId()).thenReturn(id); - Mockito.when(templateDataStoreDao.findByTemplateZone(id, id, DataStoreRole.Image)).thenReturn(Mockito.mock(TemplateDataStoreVO.class)); - Assert.assertTrue(templateService.isSkipTemplateStoreDownload(templateVO, id)); + public void shouldDownloadTemplateToStoreTestDownloadsPrivateNoRefTemplate() { + Assert.assertTrue(templateService.shouldDownloadTemplateToStore(tmpltMock, dataStoreMock)); + } + + @Test + public void shouldDownloadTemplateToStoreTestSkipsPrivateExistingTemplate() { + Mockito.when(templateDataStoreDao.findByTemplateZone(tmpltMock.getId(), zoneScopeMock.getScopeId(), DataStoreRole.Image)).thenReturn(Mockito.mock(TemplateDataStoreVO.class)); + Assert.assertFalse(templateService.shouldDownloadTemplateToStore(tmpltMock, dataStoreMock)); } @Test @@ -159,7 +180,7 @@ public class TemplateServiceImplTest { boolean result = templateService.tryCopyingTemplateToImageStore(tmpltMock, destStoreMock); Assert.assertFalse(result); - Mockito.verify(storageOrchestrator, Mockito.never()).orchestrateTemplateCopyToImageStore(Mockito.any(), Mockito.any()); + Mockito.verify(storageOrchestrator, Mockito.never()).orchestrateTemplateCopyFromSecondaryStores(Mockito.anyLong(), Mockito.any()); } @Test @@ -167,20 +188,161 @@ public class TemplateServiceImplTest { templatesInSourceStore.put(tmpltMock.getUniqueName(), tmpltPropMock); Mockito.doReturn(null).when(templateInfoMock).getInstallPath(); + Scope scopeMock = Mockito.mock(Scope.class); + Mockito.doReturn(scopeMock).when(destStoreMock).getScope(); + Mockito.doReturn(1L).when(scopeMock).getScopeId(); + Mockito.doReturn(List.of(1L)).when(_dcDao).listAllIds(); + boolean result = templateService.tryCopyingTemplateToImageStore(tmpltMock, destStoreMock); Assert.assertFalse(result); - Mockito.verify(storageOrchestrator, Mockito.never()).orchestrateTemplateCopyToImageStore(Mockito.any(), Mockito.any()); + Mockito.verify(storageOrchestrator, Mockito.never()).orchestrateTemplateCopyFromSecondaryStores(Mockito.anyLong(), Mockito.any()); } @Test - public void tryCopyingTemplateToImageStoreTestReturnsTrueWhenTemplateExistsInAnotherStorageAndTaskWasScheduled() { - templatesInSourceStore.put(tmpltMock.getUniqueName(), tmpltPropMock); - Mockito.doReturn(new AsyncCallFuture<>()).when(storageOrchestrator).orchestrateTemplateCopyToImageStore(Mockito.any(), Mockito.any()); + public void tryCopyingTemplateToImageStoreTestReturnsTrueWhenTemplateExistsInAnotherZone() throws StorageUnavailableException, ResourceAllocationException { + Scope scopeMock = Mockito.mock(Scope.class); + Mockito.doReturn(scopeMock).when(destStoreMock).getScope(); + Mockito.doReturn(1L).when(scopeMock).getScopeId(); + Mockito.doReturn(100L).when(tmpltMock).getId(); + Mockito.doReturn("unique-name").when(tmpltMock).getUniqueName(); + Mockito.doReturn(List.of(sourceStoreMock)).when(dataStoreManagerMock).getImageStoresByZoneIds(1L); + Mockito.doReturn(null).when(templateService).listTemplate(sourceStoreMock); + Mockito.doReturn(List.of(1L, 2L)).when(_dcDao).listAllIds(); + + DataStore otherZoneStoreMock = Mockito.mock(DataStore.class); + Mockito.doReturn(List.of(otherZoneStoreMock)).when(dataStoreManagerMock).getImageStoresByZoneIds(2L); + + Map templatesInOtherZone = new HashMap<>(); + templatesInOtherZone.put("unique-name", tmpltPropMock); + Mockito.doReturn(templatesInOtherZone).when(templateService).listTemplate(otherZoneStoreMock); + + TemplateObject sourceTmplMock = Mockito.mock(TemplateObject.class); + Mockito.doReturn(sourceTmplMock).when(templateDataFactoryMock).getTemplate(100L, otherZoneStoreMock); + Mockito.doReturn("/mnt/secondary/template.qcow2").when(sourceTmplMock).getInstallPath(); + + DataCenterVO dstZoneMock = Mockito.mock(DataCenterVO.class); + Mockito.doReturn(dstZoneMock).when(_dcDao).findById(1L); + Mockito.doReturn(true).when(templateManagerMock).copy(Mockito.anyLong(), Mockito.any(), Mockito.any(), Mockito.any()); boolean result = templateService.tryCopyingTemplateToImageStore(tmpltMock, destStoreMock); Assert.assertTrue(result); - Mockito.verify(storageOrchestrator).orchestrateTemplateCopyToImageStore(Mockito.any(), Mockito.any()); + } + + @Test + public void tryCopyingTemplateToImageStoreTestReturnsFalseWhenDestinationZoneIsMissing() { + Scope scopeMock = Mockito.mock(Scope.class); + Mockito.doReturn(scopeMock).when(destStoreMock).getScope(); + Mockito.doReturn(1L).when(scopeMock).getScopeId(); + Mockito.doReturn(100L).when(tmpltMock).getId(); + Mockito.doReturn("unique-name").when(tmpltMock).getUniqueName(); + Mockito.doReturn(List.of(1L, 2L)).when(_dcDao).listAllIds(); + Mockito.doReturn(List.of()).when(dataStoreManagerMock).getImageStoresByZoneIds(1L); + + DataStore otherZoneStoreMock = Mockito.mock(DataStore.class); + Mockito.doReturn(List.of(otherZoneStoreMock)).when(dataStoreManagerMock).getImageStoresByZoneIds(2L); + + Map templates = new HashMap<>(); + templates.put("unique-name", tmpltPropMock); + Mockito.doReturn(templates).when(templateService).listTemplate(otherZoneStoreMock); + + TemplateObject sourceTmplMock = Mockito.mock(TemplateObject.class); + Mockito.doReturn(sourceTmplMock).when(templateDataFactoryMock).getTemplate(100L, otherZoneStoreMock); + Mockito.doReturn("/mnt/secondary/template.qcow2").when(sourceTmplMock).getInstallPath(); + Mockito.doReturn(null).when(_dcDao).findById(1L); + + boolean result = templateService.tryCopyingTemplateToImageStore(tmpltMock, destStoreMock); + + Assert.assertFalse(result); + } + + @Test + public void tryCopyingTemplateToImageStoreTestReturnsTrueWhenCrossZoneCopyTaskIsScheduled() throws StorageUnavailableException, ResourceAllocationException { + Scope scopeMock = Mockito.mock(Scope.class); + Mockito.doReturn(scopeMock).when(destStoreMock).getScope(); + Mockito.doReturn(1L).when(scopeMock).getScopeId(); + Mockito.doReturn(100L).when(tmpltMock).getId(); + Mockito.doReturn("unique-name").when(tmpltMock).getUniqueName(); + Mockito.doReturn(List.of(1L, 2L)).when(_dcDao).listAllIds(); + Mockito.doReturn(List.of()).when(dataStoreManagerMock).getImageStoresByZoneIds(1L); + + DataStore otherZoneStoreMock = Mockito.mock(DataStore.class); + Mockito.doReturn(List.of(otherZoneStoreMock)).when(dataStoreManagerMock).getImageStoresByZoneIds(2L); + + Map templates = new HashMap<>(); + templates.put("unique-name", tmpltPropMock); + Mockito.doReturn(templates).when(templateService).listTemplate(otherZoneStoreMock); + + TemplateObject sourceTmplMock = Mockito.mock(TemplateObject.class); + Mockito.doReturn(sourceTmplMock).when(templateDataFactoryMock).getTemplate(100L, otherZoneStoreMock); + Mockito.doReturn("/mnt/secondary/template.qcow2").when(sourceTmplMock).getInstallPath(); + Mockito.doReturn(100L).when(sourceTmplMock).getId(); + + DataStore sourceStoreMock = Mockito.mock(DataStore.class); + Scope sourceScopeMock = Mockito.mock(Scope.class); + Mockito.doReturn(sourceStoreMock).when(sourceTmplMock).getDataStore(); + + DataCenterVO dstZoneMock = Mockito.mock(DataCenterVO.class); + Mockito.doReturn(dstZoneMock).when(_dcDao).findById(1L); + VMTemplateVO templateVoMock = Mockito.mock(VMTemplateVO.class); + Mockito.doReturn(templateVoMock).when(templateDao).findById(100L); + + Mockito.doReturn(true).when(templateManagerMock).copy(Mockito.anyLong(), Mockito.any(), Mockito.any(), Mockito.any()); + + Account account = mock(Account.class); + User user = mock(User.class); + CallContext callContext = mock(CallContext.class); + + boolean result = templateService.tryCopyingTemplateToImageStore(tmpltMock, destStoreMock); + + Assert.assertTrue(result); + } + + @Test + public void tryCopyingTemplateToImageStoreTestReturnsFalseWhenTemplateNotFoundInAnyZone() { + Scope scopeMock = Mockito.mock(Scope.class); + Mockito.doReturn(scopeMock).when(destStoreMock).getScope(); + Mockito.doReturn(1L).when(scopeMock).getScopeId(); + Mockito.doReturn(List.of(1L, 2L)).when(_dcDao).listAllIds(); + Mockito.doReturn(List.of(sourceStoreMock)).when(dataStoreManagerMock).getImageStoresByZoneIds(Mockito.anyLong()); + Mockito.doReturn(null).when(templateService).listTemplate(Mockito.any()); + + boolean result = templateService.tryCopyingTemplateToImageStore(tmpltMock, destStoreMock); + + Assert.assertFalse(result); + } + + @Test + public void testFindUsableTemplateReturnsTemplateWithNonNullInstallPath() { + VMTemplateVO template = Mockito.mock(VMTemplateVO.class); + Mockito.when(template.getId()).thenReturn(10L); + Mockito.when(template.getUniqueName()).thenReturn("test-template"); + + DataStore storeWithNullPath = Mockito.mock(DataStore.class); + Mockito.when(storeWithNullPath.getName()).thenReturn("store-null"); + + DataStore storeWithValidPath = Mockito.mock(DataStore.class); + TemplateObject tmplWithNullPath = Mockito.mock(TemplateObject.class); + Mockito.when(tmplWithNullPath.getInstallPath()).thenReturn(null); + + TemplateObject tmplWithValidPath = Mockito.mock(TemplateObject.class); + Mockito.when(tmplWithValidPath.getInstallPath()).thenReturn("/mnt/secondary/template.qcow2"); + + Mockito.doReturn(tmplWithNullPath).when(templateDataFactoryMock).getTemplate(10L, storeWithNullPath); + Mockito.doReturn(tmplWithValidPath).when(templateDataFactoryMock).getTemplate(10L, storeWithValidPath); + + Map templates = new HashMap<>(); + templates.put("test-template", Mockito.mock(TemplateProp.class)); + + Mockito.doReturn(templates).when(templateService).listTemplate(storeWithNullPath); + Mockito.doReturn(templates).when(templateService).listTemplate(storeWithValidPath); + + List imageStores = List.of(storeWithNullPath, storeWithValidPath); + + TemplateObject result = templateService.findUsableTemplate(template, imageStores); + + Assert.assertNotNull(result); + Assert.assertEquals(tmplWithValidPath, result); } } diff --git a/engine/storage/integration-test/src/test/java/org/apache/cloudstack/storage/test/SnapshotTest.java b/engine/storage/integration-test/src/test/java/org/apache/cloudstack/storage/test/SnapshotTest.java index 0185c0d1934..0cdc4bc65bd 100644 --- a/engine/storage/integration-test/src/test/java/org/apache/cloudstack/storage/test/SnapshotTest.java +++ b/engine/storage/integration-test/src/test/java/org/apache/cloudstack/storage/test/SnapshotTest.java @@ -268,7 +268,7 @@ public class SnapshotTest extends CloudStackTestNGBase { to.setSize(1000L); CopyCmdAnswer answer = new CopyCmdAnswer(to); templateOnStore.processEvent(Event.CreateOnlyRequested); - templateOnStore.processEvent(Event.OperationSuccessed, answer); + templateOnStore.processEvent(Event.OperationSucceeded, answer); } diff --git a/engine/storage/integration-test/src/test/java/org/apache/cloudstack/storage/test/VolumeTest.java b/engine/storage/integration-test/src/test/java/org/apache/cloudstack/storage/test/VolumeTest.java index 86421742f83..a2266020047 100644 --- a/engine/storage/integration-test/src/test/java/org/apache/cloudstack/storage/test/VolumeTest.java +++ b/engine/storage/integration-test/src/test/java/org/apache/cloudstack/storage/test/VolumeTest.java @@ -244,7 +244,7 @@ public class VolumeTest extends CloudStackTestNGBase { to.setSize(100L); CopyCmdAnswer answer = new CopyCmdAnswer(to); templateOnStore.processEvent(Event.CreateOnlyRequested); - templateOnStore.processEvent(Event.OperationSuccessed, answer); + templateOnStore.processEvent(Event.OperationSucceeded, answer); } diff --git a/engine/storage/integration-test/src/test/java/org/apache/cloudstack/storage/test/VolumeTestVmware.java b/engine/storage/integration-test/src/test/java/org/apache/cloudstack/storage/test/VolumeTestVmware.java index 98af170356f..d55b67f7a30 100644 --- a/engine/storage/integration-test/src/test/java/org/apache/cloudstack/storage/test/VolumeTestVmware.java +++ b/engine/storage/integration-test/src/test/java/org/apache/cloudstack/storage/test/VolumeTestVmware.java @@ -246,7 +246,7 @@ public class VolumeTestVmware extends CloudStackTestNGBase { to.setPath(this.getImageInstallPath()); CopyCmdAnswer answer = new CopyCmdAnswer(to); templateOnStore.processEvent(Event.CreateOnlyRequested); - templateOnStore.processEvent(Event.OperationSuccessed, answer); + templateOnStore.processEvent(Event.OperationSucceeded, answer); } diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategy.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategy.java index 1a3ee5f2325..c8c6046828a 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategy.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategy.java @@ -270,7 +270,7 @@ public class DefaultSnapshotStrategy extends SnapshotStrategyBase { } if (Snapshot.State.Error.equals(snapshotVO.getState())) { - List storeRefs = snapshotStoreDao.findBySnapshotId(snapshotId); + List storeRefs = snapshotStoreDao.findBySnapshotIdWithNonDestroyedState(snapshotId); List deletedRefs = new ArrayList<>(); for (SnapshotDataStoreVO ref : storeRefs) { boolean refZoneIdMatch = false; @@ -351,7 +351,7 @@ public class DefaultSnapshotStrategy extends SnapshotStrategyBase { protected Boolean deleteSnapshotInfo(SnapshotInfo snapshotInfo, SnapshotVO snapshotVo) { DataStore dataStore = snapshotInfo.getDataStore(); String storageToString = String.format("%s {uuid: \"%s\", name: \"%s\"}", dataStore.getRole().name(), dataStore.getUuid(), dataStore.getName()); - List snapshotStoreRefs = snapshotStoreDao.findBySnapshotId(snapshotVo.getId()); + List snapshotStoreRefs = snapshotStoreDao.findBySnapshotIdWithNonDestroyedState(snapshotVo.getId()); boolean isLastSnapshotRef = CollectionUtils.isEmpty(snapshotStoreRefs) || snapshotStoreRefs.size() == 1; try { SnapshotObject snapshotObject = castSnapshotInfoToSnapshotObject(snapshotInfo); diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotDataFactoryImpl.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotDataFactoryImpl.java index cd314b0be63..5c0a613d82d 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotDataFactoryImpl.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotDataFactoryImpl.java @@ -94,7 +94,7 @@ public class SnapshotDataFactoryImpl implements SnapshotDataFactory { if (snapshot == null) { //snapshot may have been removed; return new ArrayList<>(); } - List allSnapshotsAndDataStore = snapshotStoreDao.findBySnapshotId(snapshotId); + List allSnapshotsAndDataStore = snapshotStoreDao.findBySnapshotIdWithNonDestroyedState(snapshotId); if (CollectionUtils.isEmpty(allSnapshotsAndDataStore)) { return new ArrayList<>(); } @@ -118,7 +118,23 @@ public class SnapshotDataFactoryImpl implements SnapshotDataFactory { if (snapshot == null) { return null; } - SnapshotDataStoreVO snapshotStore = snapshotStoreDao.findByStoreSnapshot(role, storeId, snapshotId); + return getSnapshotOnStore(snapshot, storeId, role); + } + + @Override + public SnapshotInfo getSnapshotIncludingRemoved(long snapshotId, long storeId, DataStoreRole role) { + SnapshotVO snapshot = snapshotDao.findByIdIncludingRemoved(snapshotId); + if (snapshot == null) { + return null; + } + return getSnapshotOnStore(snapshot, storeId, role); + } + + private SnapshotInfo getSnapshotOnStore(SnapshotVO snapshot, long storeId, DataStoreRole role) { + if (snapshot == null) { + return null; + } + SnapshotDataStoreVO snapshotStore = snapshotStoreDao.findByStoreSnapshot(role, storeId, snapshot.getId()); if (snapshotStore == null) { return null; } @@ -207,7 +223,7 @@ public class SnapshotDataFactoryImpl implements SnapshotDataFactory { @Override public void updateOperationFailed(long snapshotId) throws NoTransitionException { - List snapshotStoreRefs = snapshotStoreDao.findBySnapshotId(snapshotId); + List snapshotStoreRefs = snapshotStoreDao.findBySnapshotIdWithNonDestroyedState(snapshotId); for (SnapshotDataStoreVO snapshotStoreRef : snapshotStoreRefs) { SnapshotInfo snapshotInfo = getSnapshot(snapshotStoreRef.getSnapshotId(), snapshotStoreRef.getDataStoreId(), snapshotStoreRef.getRole()); if (snapshotInfo != null) { diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java index de095f58dd1..5042882f553 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java @@ -382,8 +382,7 @@ public class SnapshotServiceImpl implements SnapshotService { if (res.isFailed()) { throw new CloudRuntimeException(res.getResult()); } - SnapshotInfo destSnapshot = res.getSnapshot(); - return destSnapshot; + return res.getSnapshot(); } catch (InterruptedException e) { logger.debug("failed copy snapshot", e); throw new CloudRuntimeException("Failed to copy snapshot", e); @@ -391,7 +390,6 @@ public class SnapshotServiceImpl implements SnapshotService { logger.debug("Failed to copy snapshot", e); throw new CloudRuntimeException("Failed to copy snapshot", e); } - } protected Void copySnapshotAsyncCallback(AsyncCallbackDispatcher callback, CopySnapshotContext context) { @@ -479,7 +477,6 @@ public class SnapshotServiceImpl implements SnapshotService { } protected Void deleteSnapshotCallback(AsyncCallbackDispatcher callback, DeleteSnapshotContext context) { - CommandResult result = callback.getResult(); AsyncCallFuture future = context.future; SnapshotInfo snapshot = context.snapshot; @@ -607,7 +604,7 @@ public class SnapshotServiceImpl implements SnapshotService { if (snapshot != null) { if (snapshot.getState() != Snapshot.State.BackedUp) { - List snapshotDataStoreVOs = _snapshotStoreDao.findBySnapshotId(snapshotId); + List snapshotDataStoreVOs = _snapshotStoreDao.findBySnapshotIdWithNonDestroyedState(snapshotId); for (SnapshotDataStoreVO snapshotDataStoreVO : snapshotDataStoreVOs) { logger.debug("Remove snapshot {}, status {} on snapshot_store_ref table with id: {}", snapshot, snapshotDataStoreVO.getState(), snapshotDataStoreVO.getId()); @@ -712,7 +709,6 @@ public class SnapshotServiceImpl implements SnapshotService { SnapshotObject srcSnapshot = (SnapshotObject)snapshot; srcSnapshot.processEvent(Event.DestroyRequested); srcSnapshot.processEvent(Event.OperationSucceeded); - srcSnapshot.processEvent(Snapshot.Event.OperationFailed); _snapshotDetailsDao.removeDetail(srcSnapshot.getId(), AsyncJob.Constants.MS_ID); @@ -723,7 +719,6 @@ public class SnapshotServiceImpl implements SnapshotService { } } }); - } @Override diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/StorageSystemSnapshotStrategy.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/StorageSystemSnapshotStrategy.java index 280d3bfbeb5..2bfa3eda11b 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/StorageSystemSnapshotStrategy.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/StorageSystemSnapshotStrategy.java @@ -540,7 +540,6 @@ public class StorageSystemSnapshotStrategy extends SnapshotStrategyBase { logger.warn("Failed to clean up snapshot '" + snapshot.getId() + "' on primary storage: " + e.getMessage()); } } - } private VMSnapshot takeHypervisorSnapshot(VolumeInfo volumeInfo) { diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java index 2986f3f8fab..cc2e2da3dda 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java @@ -101,6 +101,9 @@ public class ObjectInDataStoreManagerImpl implements ObjectInDataStoreManager { stateMachines.addTransition(State.Destroying, Event.DestroyRequested, State.Destroying); stateMachines.addTransition(State.Destroying, Event.OperationSucceeded, State.Destroyed); stateMachines.addTransition(State.Destroying, Event.OperationFailed, State.Destroying); + stateMachines.addTransition(State.Destroyed, Event.DestroyRequested, State.Destroyed); + stateMachines.addTransition(State.Destroyed, Event.OperationSucceeded, State.Destroyed); + stateMachines.addTransition(State.Destroyed, Event.OperationFailed, State.Destroyed); stateMachines.addTransition(State.Failed, Event.DestroyRequested, State.Destroying); // TODO: further investigate why an extra event is sent when it is // already Ready for DownloadListener diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java index 6a10c26cc0b..d864bf8cd8c 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/PrimaryDataStoreImpl.java @@ -126,7 +126,7 @@ public class PrimaryDataStoreImpl implements PrimaryDataStore { @Override public List getVolumes() { - List volumes = volumeDao.findByPoolId(getId()); + List volumes = volumeDao.findNonDestroyedVolumesByPoolId(getId()); List volumeInfos = new ArrayList(); for (VolumeVO volume : volumes) { volumeInfos.add(VolumeObject.getVolumeObject(this, volume)); diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java index eeac55be141..2c70e48706d 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java @@ -704,7 +704,7 @@ public class VolumeServiceImpl implements VolumeService { VolumeApiResult res = new VolumeApiResult(volumeInfo); if (result.isSuccess()) { - // volumeInfo.processEvent(Event.OperationSuccessed, result.getAnswer()); + // volumeInfo.processEvent(Event.OperationSucceeded, result.getAnswer()); VolumeVO volume = volDao.findById(volumeInfo.getId()); CopyCmdAnswer answer = (CopyCmdAnswer)result.getAnswer(); diff --git a/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java b/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java index 00cf56345c8..27b04ddf893 100644 --- a/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java +++ b/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java @@ -120,10 +120,18 @@ public class ConfigKey { static ConfigDepotImpl s_depot = null; - static public void init(ConfigDepotImpl depot) { + private String _defaultValueIfEmpty = null; + + public static void init(ConfigDepotImpl depot) { s_depot = depot; } + public ConfigKey(Class type, String name, String category, String defaultValue, String description, boolean isDynamic, Scope scope, T multiplier, + String displayText, String parent, Ternary group, Pair subGroup, Kind kind, String options, String defaultValueIfEmpty) { + this(type, name, category, defaultValue, description, isDynamic, scope, multiplier, displayText, parent, group, subGroup, kind, options); + this._defaultValueIfEmpty = defaultValueIfEmpty; + } + public ConfigKey(String category, Class type, String name, String defaultValue, String description, boolean isDynamic, Scope scope) { this(type, name, category, defaultValue, description, isDynamic, scope, null); } @@ -216,7 +224,19 @@ public class ConfigKey { public T value() { if (_value == null || isDynamic()) { String value = s_depot != null ? s_depot.getConfigStringValue(_name, Scope.Global, null) : null; - _value = valueOf((value == null) ? defaultValue() : value); + + String effective; + if (value != null) { + if (value.isEmpty() && _defaultValueIfEmpty != null) { + effective = _defaultValueIfEmpty; + } else { + effective = value; + } + } else { + effective = _defaultValueIfEmpty != null ? _defaultValueIfEmpty : defaultValue(); + } + + _value = valueOf(effective); } return _value; @@ -231,6 +251,10 @@ public class ConfigKey { if (value == null) { return value(); } + + if (value.isEmpty() && _defaultValueIfEmpty != null) { + return valueOf(_defaultValueIfEmpty); + } return valueOf(value); } diff --git a/framework/db/src/main/java/com/cloud/utils/db/Filter.java b/framework/db/src/main/java/com/cloud/utils/db/Filter.java index fb8c9ee37fc..90e42952a99 100644 --- a/framework/db/src/main/java/com/cloud/utils/db/Filter.java +++ b/framework/db/src/main/java/com/cloud/utils/db/Filter.java @@ -57,7 +57,8 @@ public class Filter { } public Filter(long limit) { - _orderBy = " ORDER BY RAND() LIMIT " + limit; + _orderBy = " ORDER BY RAND()"; + _limit = limit; } public Filter(Long offset, Long limit) { diff --git a/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java b/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java index 301803aab9b..535fa032d23 100644 --- a/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java +++ b/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java @@ -89,6 +89,7 @@ import net.sf.cglib.proxy.NoOp; import net.sf.ehcache.Cache; import net.sf.ehcache.CacheManager; import net.sf.ehcache.Element; +import org.springframework.util.ClassUtils; /** * GenericDaoBase is a simple way to implement DAOs. It DOES NOT @@ -1160,6 +1161,8 @@ public abstract class GenericDaoBase extends Compone if (filter.getLimit() != null) { sql.append(", ").append(filter.getLimit()); } + } else if (filter.getLimit() != null) { + sql.append(" LIMIT ").append(filter.getLimit()); } } } @@ -1321,7 +1324,7 @@ public abstract class GenericDaoBase extends Compone Filter filter = null; final long batchSizeFinal = ObjectUtils.defaultIfNull(batchSize, 0L); if (batchSizeFinal > 0) { - filter = new Filter(batchSizeFinal); + filter = new Filter(null, batchSizeFinal); } int expunged = 0; int currentExpunged = 0; @@ -2047,16 +2050,22 @@ public abstract class GenericDaoBase extends Compone @DB() protected void setField(final Object entity, final ResultSet rs, ResultSetMetaData meta, final int index) throws SQLException { - Attribute attr = _allColumns.get(new Pair(meta.getTableName(index), meta.getColumnName(index))); + String tableName = meta.getTableName(index); + String columnName = meta.getColumnName(index); + Attribute attr = _allColumns.get(new Pair<>(tableName, columnName)); if (attr == null) { // work around for mysql bug to return original table name instead of view name in db view case Table tbl = entity.getClass().getSuperclass().getAnnotation(Table.class); if (tbl != null) { - attr = _allColumns.get(new Pair(tbl.name(), meta.getColumnLabel(index))); + attr = _allColumns.get(new Pair<>(tbl.name(), meta.getColumnLabel(index))); } } - assert (attr != null) : "How come I can't find " + meta.getCatalogName(index) + "." + meta.getColumnName(index); - setField(entity, attr.field, rs, index); + if(attr == null) { + logger.warn(String.format("Failed to find attribute in the entity %s to map column %s.%s (%s)", + ClassUtils.getUserClass(entity).getSimpleName(), tableName, columnName)); + } else { + setField(entity, attr.field, rs, index); + } } @Override diff --git a/packaging/el8/cloud.spec b/packaging/el8/cloud.spec index 3d485112266..507a6e64173 100644 --- a/packaging/el8/cloud.spec +++ b/packaging/el8/cloud.spec @@ -76,7 +76,7 @@ Requires: sudo Requires: /sbin/service Requires: /sbin/chkconfig Requires: /usr/bin/ssh-keygen -Requires: (genisoimage or mkisofs) +Requires: (genisoimage or mkisofs or xorrisofs) Requires: ipmitool Requires: %{name}-common = %{_ver} Requires: (iptables-services or iptables) diff --git a/plugins/api/vmware-sioc/src/main/java/org/apache/cloudstack/sioc/SiocManagerImpl.java b/plugins/api/vmware-sioc/src/main/java/org/apache/cloudstack/sioc/SiocManagerImpl.java index e93b8df39e9..b01af35725f 100644 --- a/plugins/api/vmware-sioc/src/main/java/org/apache/cloudstack/sioc/SiocManagerImpl.java +++ b/plugins/api/vmware-sioc/src/main/java/org/apache/cloudstack/sioc/SiocManagerImpl.java @@ -123,7 +123,7 @@ public class SiocManagerImpl implements SiocManager { int limitIopsTotal = 0; - List volumes = volumeDao.findByPoolId(storagePoolId, null); + List volumes = volumeDao.findNonDestroyedVolumesByPoolId(storagePoolId, null); if (volumes != null && volumes.size() > 0) { Set instanceIds = new HashSet<>(); diff --git a/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalDnsmasqResource.java b/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalDnsmasqResource.java index 51acfe93d39..8e7efedfca3 100644 --- a/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalDnsmasqResource.java +++ b/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalDnsmasqResource.java @@ -46,10 +46,10 @@ public class BaremetalDnsmasqResource extends BaremetalDhcpResourceBase { com.trilead.ssh2.Connection sshConnection = null; try { super.configure(name, params); - logger.debug(String.format("Trying to connect to DHCP server(IP=%1$s, username=%2$s, password=%3$s)", _ip, _username, _password)); + logger.debug(String.format("Trying to connect to DHCP server(IP=%1$s, username=%2$s", _ip, _username)); sshConnection = SSHCmdHelper.acquireAuthorizedConnection(_ip, _username, _password); if (sshConnection == null) { - throw new ConfigurationException(String.format("Cannot connect to DHCP server(IP=%1$s, username=%2$s, password=%3$s", _ip, _username, _password)); + throw new ConfigurationException(String.format("Cannot connect to DHCP server(IP=%1$s, username=%2$s", _ip, _username)); } if (!SSHCmdHelper.sshExecuteCmd(sshConnection, "[ -f '/usr/sbin/dnsmasq' ]")) { diff --git a/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalKickStartPxeResource.java b/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalKickStartPxeResource.java index 3775f4effc1..88c4dea96b3 100644 --- a/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalKickStartPxeResource.java +++ b/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalKickStartPxeResource.java @@ -130,8 +130,8 @@ public class BaremetalKickStartPxeResource extends BaremetalPxeResourceBase { sshConnection.connect(null, 60000, 60000); if (!sshConnection.authenticateWithPassword(_username, _password)) { - logger.debug("SSH Failed to authenticate"); - throw new ConfigurationException(String.format("Cannot connect to PING PXE server(IP=%1$s, username=%2$s, password=%3$s", _ip, _username, _password)); + logger.debug("SSH Failed to authenticate with user {} credentials", _username); + throw new ConfigurationException(String.format("Cannot connect to PING PXE server(IP=%1$s, username=%2$s", _ip, _username)); } String script = String.format("python /usr/bin/baremetal_user_data.py '%s'", arg); @@ -167,7 +167,7 @@ public class BaremetalKickStartPxeResource extends BaremetalPxeResourceBase { sshConnection.connect(null, 60000, 60000); if (!sshConnection.authenticateWithPassword(_username, _password)) { logger.debug("SSH Failed to authenticate"); - throw new ConfigurationException(String.format("Cannot connect to PING PXE server(IP=%1$s, username=%2$s, password=%3$s", _ip, _username, _password)); + throw new ConfigurationException(String.format("Cannot connect to PING PXE server(IP=%1$s, username=%2$s", _ip, _username)); } String copyTo = String.format("%s/%s", _tftpDir, cmd.getTemplateUuid()); diff --git a/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalPingPxeResource.java b/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalPingPxeResource.java index 96b2dbfeb93..a54cd4a1a11 100644 --- a/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalPingPxeResource.java +++ b/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/networkservice/BaremetalPingPxeResource.java @@ -101,7 +101,7 @@ public class BaremetalPingPxeResource extends BaremetalPxeResourceBase { sshConnection.connect(null, 60000, 60000); if (!sshConnection.authenticateWithPassword(_username, _password)) { logger.debug("SSH Failed to authenticate"); - throw new ConfigurationException(String.format("Cannot connect to PING PXE server(IP=%1$s, username=%2$s, password=%3$s", _ip, _username, "******")); + throw new ConfigurationException(String.format("Cannot connect to PING PXE server(IP=%1$s, username=%2$s, password=******", _ip, _username)); } String cmd = String.format("[ -f /%1$s/pxelinux.0 ] && [ -f /%2$s/kernel ] && [ -f /%3$s/initrd.gz ] ", _tftpDir, _tftpDir, _tftpDir); @@ -150,8 +150,8 @@ public class BaremetalPingPxeResource extends BaremetalPxeResourceBase { try { sshConnection.connect(null, 60000, 60000); if (!sshConnection.authenticateWithPassword(_username, _password)) { - logger.debug("SSH Failed to authenticate"); - throw new ConfigurationException(String.format("Cannot connect to PING PXE server(IP=%1$s, username=%2$s, password=%3$s", _ip, _username, _password)); + logger.debug("SSH Failed to authenticate with user {} credentials", _username); + throw new ConfigurationException(String.format("Cannot connect to PING PXE server(IP=%1$s, username=%2$s", _ip, _username)); } String script = @@ -179,7 +179,7 @@ public class BaremetalPingPxeResource extends BaremetalPxeResourceBase { sshConnection.connect(null, 60000, 60000); if (!sshConnection.authenticateWithPassword(_username, _password)) { logger.debug("SSH Failed to authenticate"); - throw new ConfigurationException(String.format("Cannot connect to PING PXE server(IP=%1$s, username=%2$s, password=%3$s", _ip, _username, _password)); + throw new ConfigurationException(String.format("Cannot connect to PING PXE server(IP=%1$s, username=%2$s", _ip, _username)); } String script = @@ -237,7 +237,7 @@ public class BaremetalPingPxeResource extends BaremetalPxeResourceBase { sshConnection.connect(null, 60000, 60000); if (!sshConnection.authenticateWithPassword(_username, _password)) { logger.debug("SSH Failed to authenticate"); - throw new ConfigurationException(String.format("Cannot connect to PING PXE server(IP=%1$s, username=%2$s, password=%3$s", _ip, _username, _password)); + throw new ConfigurationException(String.format("Cannot connect to PING PXE server(IP=%1$s, username=%2$s", _ip, _username)); } String script = String.format("python /usr/bin/baremetal_user_data.py '%s'", arg); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java index 3b66529ccaf..b4f7fbd6dac 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java @@ -232,7 +232,7 @@ public class BridgeVifDriver extends VifDriverBase { String brName = createVnetBr(vNetId, trafficLabel, protocol); intf.defBridgeNet(brName, null, nic.getMac(), getGuestNicModel(guestOsType, nicAdapter), networkRateKBps); } else { - String brName = createVnetBr(vNetId, _bridges.get("private"), protocol); + String brName = createVnetBr(vNetId, _bridges.get("guest"), protocol); intf.defBridgeNet(brName, null, nic.getMac(), getGuestNicModel(guestOsType, nicAdapter), networkRateKBps); } } else { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 1bbebfe3ee5..cb16cfa2fe6 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -4371,12 +4371,11 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv String dataDiskController = details.get(VmDetailConstants.DATA_DISK_CONTROLLER); if (StringUtils.isNotBlank(dataDiskController)) { - LOGGER.debug("Passed custom disk controller for DATA disk " + dataDiskController); - for (DiskDef.DiskBus bus : DiskDef.DiskBus.values()) { - if (bus.toString().equalsIgnoreCase(dataDiskController)) { - LOGGER.debug("Found matching enum for disk controller for DATA disk " + dataDiskController); - return bus; - } + LOGGER.debug("Passed custom disk controller for DATA disk {}", dataDiskController); + DiskDef.DiskBus bus = DiskDef.DiskBus.fromValue(dataDiskController); + if (bus != null) { + LOGGER.debug("Found matching enum for disk controller for DATA disk {}", dataDiskController); + return bus; } } return null; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtDomainXMLParser.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtDomainXMLParser.java index 8b26bc94e21..4d823783a99 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtDomainXMLParser.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtDomainXMLParser.java @@ -111,7 +111,9 @@ public class LibvirtDomainXMLParser { def.defNetworkBasedDisk(diskPath, host, port, authUserName, poolUuid, diskLabel, DiskDef.DiskBus.valueOf(bus.toUpperCase()), DiskDef.DiskProtocol.valueOf(protocol.toUpperCase()), fmt); - def.setCacheMode(DiskDef.DiskCacheMode.valueOf(diskCacheMode.toUpperCase())); + if (StringUtils.isNotBlank(diskCacheMode)) { + def.setCacheMode(DiskDef.DiskCacheMode.valueOf(diskCacheMode.toUpperCase())); + } } else { String diskFmtType = getAttrValue("driver", "type", disk); String diskCacheMode = getAttrValue("driver", "cache", disk); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java index a38e7a02357..696e71bea80 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java @@ -686,6 +686,15 @@ public class LibvirtVMDef { _bus = bus; } + public static DiskBus fromValue(String bus) { + for (DiskBus b : DiskBus.values()) { + if (b.toString().equalsIgnoreCase(bus)) { + return b; + } + } + return null; + } + @Override public String toString() { return _bus; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java index 32f2a4b122c..fe18a88fe1f 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java @@ -158,7 +158,7 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper dpdkPortsMapping = command.getDpdkInterfaceMapping(); if (MapUtils.isNotEmpty(dpdkPortsMapping)) { if (logger.isTraceEnabled()) { - logger.trace(String.format("Changing VM [%s] DPDK interfaces during migration to host: [%s].", vmName, target)); + logger.trace("Changing VM {} DPDK interfaces during migration to host: {}.", vmName, target); } xmlDesc = replaceDpdkInterfaces(xmlDesc, dpdkPortsMapping); if (logger.isDebugEnabled()) { - logger.debug(String.format("Changed VM [%s] XML configuration of DPDK interfaces. New XML configuration is [%s].", vmName, xmlDesc)); + logger.debug("Changed VM {} XML configuration of DPDK interfaces. New XML configuration is {}.", vmName, maskSensitiveInfoInXML(xmlDesc)); } } @@ -233,7 +233,7 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper migrateThread = executor.submit(worker); executor.shutdown(); long sleeptime = 0; + final int migrateDowntime = libvirtComputingResource.getMigrateDowntime(); + boolean isMigrateDowntimeSet = false; + while (!executor.isTerminated()) { Thread.sleep(100); sleeptime += 100; - if (sleeptime == 1000) { // wait 1s before attempting to set downtime on migration, since I don't know of a VIR_DOMAIN_MIGRATING state - final int migrateDowntime = libvirtComputingResource.getMigrateDowntime(); - if (migrateDowntime > 0 ) { - try { - final int setDowntime = dm.migrateSetMaxDowntime(migrateDowntime); - if (setDowntime == 0 ) { - logger.debug("Set max downtime for migration of " + vmName + " to " + String.valueOf(migrateDowntime) + "ms"); - } - } catch (final LibvirtException e) { - logger.debug("Failed to set max downtime for migration, perhaps migration completed? Error: " + e.getMessage()); + if (!isMigrateDowntimeSet && migrateDowntime > 0 && sleeptime >= 1000) { // wait 1s before attempting to set downtime on migration, since I don't know of a VIR_DOMAIN_MIGRATING state + try { + final int setDowntime = dm.migrateSetMaxDowntime(migrateDowntime); + if (setDowntime == 0 ) { + isMigrateDowntimeSet = true; + logger.debug("Set max downtime for migration of " + vmName + " to " + String.valueOf(migrateDowntime) + "ms"); } + } catch (final LibvirtException e) { + logger.debug("Failed to set max downtime for migration, perhaps migration completed? Error: " + e.getMessage()); } } if (sleeptime % 1000 == 0) { @@ -272,7 +273,7 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper]*type=['\"]vnc['\"][^>]*passwd=['\"])([^'\"]*)(['\"])", + "$1*****$3"); + } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartCommandWrapper.java index a174c9a6f14..6e978715755 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtStartCommandWrapper.java @@ -80,8 +80,9 @@ public final class LibvirtStartCommandWrapper extends CommandWrapper details) + final Long iopsWriteRateMaxLength, final String cacheMode, final DiskDef.LibvirtDiskEncryptDetails encryptDetails, Map details, Map controllerInfo) throws LibvirtException, InternalErrorException { attachOrDetachDisk(conn, attach, vmName, attachingDisk, devId, serial, bytesReadRate, bytesReadRateMax, bytesReadRateMaxLength, bytesWriteRate, bytesWriteRateMax, bytesWriteRateMaxLength, iopsReadRate, iopsReadRateMax, iopsReadRateMaxLength, iopsWriteRate, - iopsWriteRateMax, iopsWriteRateMaxLength, cacheMode, encryptDetails, 0l, details); + iopsWriteRateMax, iopsWriteRateMaxLength, cacheMode, encryptDetails, 0l, details, controllerInfo); } /** * * Attaches or detaches a disk to an instance. - * @param conn libvirt connection - * @param attach boolean that determines whether the device will be attached or detached - * @param vmName instance name - * @param attachingDisk kvm physical disk - * @param devId device id in instance + * @param conn libvirt connection + * @param attach boolean that determines whether the device will be attached or detached + * @param vmName instance name + * @param attachingDisk kvm physical disk + * @param devId device id in instance * @param serial - * @param bytesReadRate bytes read rate - * @param bytesReadRateMax bytes read rate max - * @param bytesReadRateMaxLength bytes read rate max length - * @param bytesWriteRate bytes write rate - * @param bytesWriteRateMax bytes write rate amx + * @param bytesReadRate bytes read rate + * @param bytesReadRateMax bytes read rate max + * @param bytesReadRateMaxLength bytes read rate max length + * @param bytesWriteRate bytes write rate + * @param bytesWriteRateMax bytes write rate amx * @param bytesWriteRateMaxLength bytes write rate max length - * @param iopsReadRate iops read rate - * @param iopsReadRateMax iops read rate max - * @param iopsReadRateMaxLength iops read rate max length - * @param iopsWriteRate iops write rate - * @param iopsWriteRateMax iops write rate max - * @param iopsWriteRateMaxLength iops write rate max length - * @param cacheMode cache mode - * @param encryptDetails encrypt details - * @param waitDetachDevice value set in milliseconds to wait before assuming device removal failed + * @param iopsReadRate iops read rate + * @param iopsReadRateMax iops read rate max + * @param iopsReadRateMaxLength iops read rate max length + * @param iopsWriteRate iops write rate + * @param iopsWriteRateMax iops write rate max + * @param iopsWriteRateMaxLength iops write rate max length + * @param cacheMode cache mode + * @param encryptDetails encrypt details + * @param waitDetachDevice value set in milliseconds to wait before assuming device removal failed + * @param controllerInfo * @throws LibvirtException * @throws InternalErrorException */ @@ -1386,7 +1388,7 @@ public class KVMStorageProcessor implements StorageProcessor { final Long bytesWriteRate, final Long bytesWriteRateMax, final Long bytesWriteRateMaxLength, final Long iopsReadRate, final Long iopsReadRateMax, final Long iopsReadRateMaxLength, final Long iopsWriteRate, final Long iopsWriteRateMax, final Long iopsWriteRateMaxLength, final String cacheMode, final DiskDef.LibvirtDiskEncryptDetails encryptDetails, - long waitDetachDevice, Map details) + long waitDetachDevice, Map details, Map controllerInfo) throws LibvirtException, InternalErrorException { List disks = null; @@ -1423,17 +1425,7 @@ public class KVMStorageProcessor implements StorageProcessor { return; } } else { - DiskDef.DiskBus busT = DiskDef.DiskBus.VIRTIO; - for (final DiskDef disk : disks) { - if (disk.getDeviceType() == DeviceType.DISK) { - if (disk.getBusType() == DiskDef.DiskBus.SCSI) { - busT = DiskDef.DiskBus.SCSI; - } else if (disk.getBusType() == DiskDef.DiskBus.VIRTIOBLK) { - busT = DiskDef.DiskBus.VIRTIOBLK; - } - break; - } - } + DiskDef.DiskBus busT = getAttachDiskBusType(devId, disks, controllerInfo); diskdef = new DiskDef(); if (busT == DiskDef.DiskBus.SCSI || busT == DiskDef.DiskBus.VIRTIOBLK) { diskdef.setQemuDriver(true); @@ -1538,6 +1530,28 @@ public class KVMStorageProcessor implements StorageProcessor { } } + protected DiskDef.DiskBus getAttachDiskBusType(int deviceId, List disks, Map controllerInfo) { + String controllerKey = deviceId == 0 ? VmDetailConstants.ROOT_DISK_CONTROLLER : VmDetailConstants.DATA_DISK_CONTROLLER; + String diskController = MapUtils.getString(controllerInfo, controllerKey); + DiskDef.DiskBus busType = DiskDef.DiskBus.fromValue(diskController); + if (diskController != null) { + logger.debug("Using controller '{}' from command specified as {} while attaching disk (deviceId={})", + diskController, controllerKey, deviceId); + return busType; + } + for (final DiskDef disk : disks) { + if (disk.getDeviceType() != DeviceType.DISK) { + continue; + } + if (disk.getBusType() == DiskDef.DiskBus.SCSI) { + return DiskDef.DiskBus.SCSI; + } else if (disk.getBusType() == DiskDef.DiskBus.VIRTIOBLK) { + return DiskDef.DiskBus.VIRTIOBLK; + } + } + return DiskDef.DiskBus.VIRTIO; + } + @Override public Answer attachVolume(final AttachCommand cmd) { final DiskTO disk = cmd.getDisk(); @@ -1565,7 +1579,8 @@ public class KVMStorageProcessor implements StorageProcessor { vol.getBytesReadRate(), vol.getBytesReadRateMax(), vol.getBytesReadRateMaxLength(), vol.getBytesWriteRate(), vol.getBytesWriteRateMax(), vol.getBytesWriteRateMaxLength(), vol.getIopsReadRate(), vol.getIopsReadRateMax(), vol.getIopsReadRateMaxLength(), - vol.getIopsWriteRate(), vol.getIopsWriteRateMax(), vol.getIopsWriteRateMaxLength(), volCacheMode, encryptDetails, disk.getDetails()); + vol.getIopsWriteRate(), vol.getIopsWriteRateMax(), vol.getIopsWriteRateMaxLength(), volCacheMode, + encryptDetails, disk.getDetails(), cmd.getControllerInfo()); return new AttachAnswer(disk); } catch (final LibvirtException e) { @@ -1602,7 +1617,7 @@ public class KVMStorageProcessor implements StorageProcessor { vol.getBytesReadRate(), vol.getBytesReadRateMax(), vol.getBytesReadRateMaxLength(), vol.getBytesWriteRate(), vol.getBytesWriteRateMax(), vol.getBytesWriteRateMaxLength(), vol.getIopsReadRate(), vol.getIopsReadRateMax(), vol.getIopsReadRateMaxLength(), - vol.getIopsWriteRate(), vol.getIopsWriteRateMax(), vol.getIopsWriteRateMaxLength(), volCacheMode, null, waitDetachDevice, null); + vol.getIopsWriteRate(), vol.getIopsWriteRateMax(), vol.getIopsWriteRateMaxLength(), volCacheMode, null, waitDetachDevice, null, null); storagePoolMgr.disconnectPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), vol.getPath()); diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapperTest.java index 3c5e54e2ba8..05d89cc3d97 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapperTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapperTest.java @@ -589,7 +589,7 @@ public class LibvirtMigrateCommandWrapperTest { @Test public void testReplaceIpForVNCInDescFile() { final String targetIp = "192.168.22.21"; - final String result = libvirtMigrateCmdWrapper.replaceIpForVNCInDescFileAndNormalizePassword(fullfile, targetIp, null, ""); + final String result = libvirtMigrateCmdWrapper.replaceIpForVNCInDescFileAndNormalizePassword(fullfile, targetIp, "vncSecretPwd", ""); assertEquals("transformation does not live up to expectation:\n" + result, targetfile, result); } @@ -1019,4 +1019,28 @@ public class LibvirtMigrateCommandWrapperTest { Assert.assertTrue(finalXml.contains(newIsoVolumePath)); } + + @Test + public void testMaskVncPwdDomain() { + // Test case 1: Single quotes + String xml1 = ""; + String expected1 = ""; + assertEquals(expected1, LibvirtMigrateCommandWrapper.maskSensitiveInfoInXML(xml1)); + + // Test case 2: Double quotes + String xml2 = ""; + String expected2 = ""; + assertEquals(expected2, LibvirtMigrateCommandWrapper.maskSensitiveInfoInXML(xml2)); + + // Test case 3: Non-VNC graphics (should remain unchanged) + String xml3 = ""; + assertEquals(xml3, LibvirtMigrateCommandWrapper.maskSensitiveInfoInXML(xml3)); + + // Test case 4: Multiple VNC entries in one string + String xml4 = "\n" + + ""; + String expected4 = "\n" + + ""; + assertEquals(expected4, LibvirtMigrateCommandWrapper.maskSensitiveInfoInXML(xml4)); + } } diff --git a/plugins/hypervisors/ovm/src/main/java/com/cloud/ovm/hypervisor/OvmResourceBase.java b/plugins/hypervisors/ovm/src/main/java/com/cloud/ovm/hypervisor/OvmResourceBase.java index 9d958a9894a..a65e4d778d3 100644 --- a/plugins/hypervisors/ovm/src/main/java/com/cloud/ovm/hypervisor/OvmResourceBase.java +++ b/plugins/hypervisors/ovm/src/main/java/com/cloud/ovm/hypervisor/OvmResourceBase.java @@ -362,7 +362,7 @@ public class OvmResourceBase implements ServerResource, HypervisorResource { sshConnection = SSHCmdHelper.acquireAuthorizedConnection(_ip, _username, _password); if (sshConnection == null) { - throw new CloudRuntimeException(String.format("Cannot connect to ovm host(IP=%1$s, username=%2$s, password=%3$s", _ip, _username, _password)); + throw new CloudRuntimeException(String.format("Cannot connect to ovm host(IP=%1$s, username=%2$s)", _ip, _username)); } if (!SSHCmdHelper.sshExecuteCmd(sshConnection, "sh /usr/bin/configureOvm.sh postSetup")) { diff --git a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixUpdateHostPasswordCommandWrapper.java b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixUpdateHostPasswordCommandWrapper.java index 1acc292b450..e3ee0ca13ca 100644 --- a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixUpdateHostPasswordCommandWrapper.java +++ b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixUpdateHostPasswordCommandWrapper.java @@ -45,9 +45,9 @@ public final class CitrixUpdateHostPasswordCommandWrapper extends CommandWrapper Pair result; try { - logger.debug("Executing command in Host: " + cmdLine); + logger.debug("Executing password update command on host: {} for user: {}", hostIp, username); final String hostPassword = citrixResourceBase.getPwdFromQueue(); - result = xenServerUtilitiesHelper.executeSshWrapper(hostIp, 22, username, null, hostPassword, cmdLine.toString()); + result = xenServerUtilitiesHelper.executeSshWrapper(hostIp, 22, username, null, hostPassword, cmdLine); } catch (final Exception e) { return new Answer(command, false, e.getMessage()); } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index 07c7f6eb547..0b211f2f235 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -973,9 +973,12 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne CallContext networkContext = CallContext.register(CallContext.current(), ApiCommandResourceType.Network); try { + Long zoneId = zone.getId(); + Integer publicMTU = NetworkService.VRPublicInterfaceMtu.valueIn(zoneId); + Integer privateMTU = NetworkService.VRPrivateInterfaceMtu.valueIn(zoneId); network = networkService.createGuestNetwork(networkOffering.getId(), clusterName + "-network", - owner.getAccountName() + "-network", owner, physicalNetwork, zone.getId(), - ControlledEntity.ACLType.Account); + owner.getAccountName() + "-network", owner, physicalNetwork, zoneId, + ControlledEntity.ACLType.Account, new Pair<>(publicMTU, privateMTU)); } catch (ConcurrentOperationException | InsufficientCapacityException | ResourceAllocationException e) { logAndThrow(Level.ERROR, String.format("Unable to create network for the Kubernetes cluster: %s", clusterName)); } finally { diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java index 99c9a4de051..8363f6f87e3 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/version/KubernetesVersionManagerImpl.java @@ -33,6 +33,9 @@ import org.apache.cloudstack.api.command.user.kubernetes.version.ListKubernetesS import org.apache.cloudstack.api.response.KubernetesSupportedVersionResponse; import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; +import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import com.cloud.api.query.dao.TemplateJoinDao; @@ -53,6 +56,7 @@ import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VMTemplateZoneDao; import com.cloud.template.TemplateApiService; import com.cloud.template.VirtualMachineTemplate; +import com.cloud.user.Account; import com.cloud.user.AccountManager; import com.cloud.utils.Pair; import com.cloud.utils.component.ComponentContext; @@ -80,12 +84,14 @@ public class KubernetesVersionManagerImpl extends ManagerBase implements Kuberne @Inject private DataCenterDao dataCenterDao; @Inject + private ImageStoreDao imageStoreDao; + @Inject private TemplateApiService templateService; public static final String MINIMUN_AUTOSCALER_SUPPORTED_VERSION = "1.15.0"; protected void updateTemplateDetailsInKubernetesSupportedVersionResponse( - final KubernetesSupportedVersion kubernetesSupportedVersion, KubernetesSupportedVersionResponse response) { + final KubernetesSupportedVersion kubernetesSupportedVersion, KubernetesSupportedVersionResponse response, boolean isRootAdmin) { TemplateJoinVO template = templateJoinDao.findById(kubernetesSupportedVersion.getIsoId()); if (template == null) { return; @@ -95,11 +101,14 @@ public class KubernetesVersionManagerImpl extends ManagerBase implements Kuberne if (template.getState() != null) { response.setIsoState(template.getState().toString()); } + if (isRootAdmin) { + response.setIsoUrl(template.getUrl()); + } response.setIsoArch(template.getArch().getType()); response.setDirectDownload(template.isDirectDownload()); } - private KubernetesSupportedVersionResponse createKubernetesSupportedVersionResponse(final KubernetesSupportedVersion kubernetesSupportedVersion) { + private KubernetesSupportedVersionResponse createKubernetesSupportedVersionResponse(final KubernetesSupportedVersion kubernetesSupportedVersion, boolean isRootAdmin) { KubernetesSupportedVersionResponse response = new KubernetesSupportedVersionResponse(); response.setObjectName("kubernetessupportedversion"); response.setId(kubernetesSupportedVersion.getUuid()); @@ -118,7 +127,7 @@ public class KubernetesVersionManagerImpl extends ManagerBase implements Kuberne response.setSupportsHA(compareSemanticVersions(kubernetesSupportedVersion.getSemanticVersion(), KubernetesClusterService.MIN_KUBERNETES_VERSION_HA_SUPPORT)>=0); response.setSupportsAutoscaling(versionSupportsAutoscaling(kubernetesSupportedVersion)); - updateTemplateDetailsInKubernetesSupportedVersionResponse(kubernetesSupportedVersion, response); + updateTemplateDetailsInKubernetesSupportedVersionResponse(kubernetesSupportedVersion, response, isRootAdmin); response.setCreated(kubernetesSupportedVersion.getCreated()); return response; } @@ -126,8 +135,11 @@ public class KubernetesVersionManagerImpl extends ManagerBase implements Kuberne private ListResponse createKubernetesSupportedVersionListResponse( List versions, Integer count) { List responseList = new ArrayList<>(); + Account caller = CallContext.current().getCallingAccount(); + boolean isRootAdmin = accountManager.isRootAdmin(caller.getId()); + for (KubernetesSupportedVersionVO version : versions) { - responseList.add(createKubernetesSupportedVersionResponse(version)); + responseList.add(createKubernetesSupportedVersionResponse(version, isRootAdmin)); } ListResponse response = new ListResponse<>(); response.setResponses(responseList, count); @@ -316,6 +328,32 @@ public class KubernetesVersionManagerImpl extends ManagerBase implements Kuberne return createKubernetesSupportedVersionListResponse(versions, versionsAndCount.second()); } + private void validateImageStoreForZone(Long zoneId, boolean directDownload) { + if (directDownload) { + return; + } + if (zoneId != null) { + List imageStores = imageStoreDao.listStoresByZoneId(zoneId); + if (CollectionUtils.isEmpty(imageStores)) { + DataCenterVO zone = dataCenterDao.findById(zoneId); + String zoneName = zone != null ? zone.getName() : String.valueOf(zoneId); + throw new InvalidParameterValueException(String.format("Unable to register Kubernetes version ISO. No image store available in zone: %s", zoneName)); + } + } else { + List zones = dataCenterDao.listAllZones(); + List zonesWithoutStorage = new ArrayList<>(); + for (DataCenterVO zone : zones) { + List imageStores = imageStoreDao.listStoresByZoneId(zone.getId()); + if (CollectionUtils.isEmpty(imageStores)) { + zonesWithoutStorage.add(zone.getName()); + } + } + if (!zonesWithoutStorage.isEmpty()) { + throw new InvalidParameterValueException(String.format("Unable to register Kubernetes version ISO for all zones. The following zones have no image store: %s", String.join(", ", zonesWithoutStorage))); + } + } + } + @Override @ActionEvent(eventType = KubernetesVersionEventTypes.EVENT_KUBERNETES_VERSION_ADD, eventDescription = "Adding Kubernetes supported version") @@ -361,6 +399,8 @@ public class KubernetesVersionManagerImpl extends ManagerBase implements Kuberne } } + validateImageStoreForZone(zoneId, isDirectDownload); + VMTemplateVO template = null; try { VirtualMachineTemplate vmTemplate = registerKubernetesVersionIso(zoneId, name, isoUrl, isoChecksum, isDirectDownload, arch); @@ -374,7 +414,7 @@ public class KubernetesVersionManagerImpl extends ManagerBase implements Kuberne supportedVersionVO = kubernetesSupportedVersionDao.persist(supportedVersionVO); CallContext.current().putContextParameter(KubernetesSupportedVersion.class, supportedVersionVO.getUuid()); - return createKubernetesSupportedVersionResponse(supportedVersionVO); + return createKubernetesSupportedVersionResponse(supportedVersionVO, true); } @Override @@ -435,7 +475,7 @@ public class KubernetesVersionManagerImpl extends ManagerBase implements Kuberne } version = kubernetesSupportedVersionDao.findById(versionId); } - return createKubernetesSupportedVersionResponse(version); + return createKubernetesSupportedVersionResponse(version, true); } @Override diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesSupportedVersionResponse.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesSupportedVersionResponse.java index cfa3212e409..f6e1ee85944 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesSupportedVersionResponse.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesSupportedVersionResponse.java @@ -50,6 +50,10 @@ public class KubernetesSupportedVersionResponse extends BaseResponse { @Param(description = "The name of the binaries ISO for Kubernetes supported version") private String isoName; + @SerializedName(ApiConstants.ISO_URL) + @Param(description = "the URL of the binaries ISO for Kubernetes supported version") + private String isoUrl; + @SerializedName(ApiConstants.ISO_STATE) @Param(description = "The state of the binaries ISO for Kubernetes supported version") private String isoState; @@ -134,6 +138,14 @@ public class KubernetesSupportedVersionResponse extends BaseResponse { this.isoName = isoName; } + public String getIsoUrl() { + return isoUrl; + } + + public void setIsoUrl(String isoUrl) { + this.isoUrl = isoUrl; + } + public String getIsoState() { return isoState; } diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/version/KubernetesVersionManagerImplTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/version/KubernetesVersionManagerImplTest.java index f827610c3cb..35f8e66e045 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/version/KubernetesVersionManagerImplTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/version/KubernetesVersionManagerImplTest.java @@ -16,10 +16,15 @@ // under the License. package com.cloud.kubernetes.version; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; import java.util.UUID; import org.apache.cloudstack.api.response.KubernetesSupportedVersionResponse; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; +import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; +import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -32,6 +37,9 @@ import org.springframework.test.util.ReflectionTestUtils; import com.cloud.api.query.dao.TemplateJoinDao; import com.cloud.api.query.vo.TemplateJoinVO; import com.cloud.cpu.CPU; +import com.cloud.dc.DataCenterVO; +import com.cloud.dc.dao.DataCenterDao; +import com.cloud.exception.InvalidParameterValueException; @RunWith(MockitoJUnitRunner.class) public class KubernetesVersionManagerImplTest { @@ -39,6 +47,12 @@ public class KubernetesVersionManagerImplTest { @Mock TemplateJoinDao templateJoinDao; + @Mock + ImageStoreDao imageStoreDao; + + @Mock + DataCenterDao dataCenterDao; + @InjectMocks KubernetesVersionManagerImpl kubernetesVersionManager = new KubernetesVersionManagerImpl(); @@ -48,7 +62,7 @@ public class KubernetesVersionManagerImplTest { Mockito.when(kubernetesSupportedVersion.getIsoId()).thenReturn(1L); KubernetesSupportedVersionResponse response = new KubernetesSupportedVersionResponse(); kubernetesVersionManager.updateTemplateDetailsInKubernetesSupportedVersionResponse(kubernetesSupportedVersion, - response); + response, true); Assert.assertNull(ReflectionTestUtils.getField(response, "isoId")); } @@ -63,13 +77,71 @@ public class KubernetesVersionManagerImplTest { Mockito.when(templateJoinVO.getUuid()).thenReturn(uuid); Mockito.when(templateJoinDao.findById(1L)).thenReturn(templateJoinVO); kubernetesVersionManager.updateTemplateDetailsInKubernetesSupportedVersionResponse(kubernetesSupportedVersion, - response); + response, true); Assert.assertEquals(uuid, ReflectionTestUtils.getField(response, "isoId")); Assert.assertNull(ReflectionTestUtils.getField(response, "isoState")); ObjectInDataStoreStateMachine.State state = ObjectInDataStoreStateMachine.State.Ready; Mockito.when(templateJoinVO.getState()).thenReturn(state); kubernetesVersionManager.updateTemplateDetailsInKubernetesSupportedVersionResponse(kubernetesSupportedVersion, - response); + response, true); Assert.assertEquals(state.toString(), ReflectionTestUtils.getField(response, "isoState")); } + + @Test + public void testValidateImageStoreForZoneWithDirectDownload() { + ReflectionTestUtils.invokeMethod(kubernetesVersionManager, "validateImageStoreForZone", 1L, true); + } + + @Test + public void testValidateImageStoreForZoneWithValidZone() { + Long zoneId = 1L; + List imageStores = Collections.singletonList(Mockito.mock(ImageStoreVO.class)); + Mockito.when(imageStoreDao.listStoresByZoneId(zoneId)).thenReturn(imageStores); + + ReflectionTestUtils.invokeMethod(kubernetesVersionManager, "validateImageStoreForZone", zoneId, false); + } + + @Test(expected = InvalidParameterValueException.class) + public void testValidateImageStoreForZoneWithNoImageStore() { + Long zoneId = 1L; + DataCenterVO zone = Mockito.mock(DataCenterVO.class); + Mockito.when(zone.getName()).thenReturn("test-zone"); + Mockito.when(dataCenterDao.findById(zoneId)).thenReturn(zone); + Mockito.when(imageStoreDao.listStoresByZoneId(zoneId)).thenReturn(Collections.emptyList()); + + ReflectionTestUtils.invokeMethod(kubernetesVersionManager, "validateImageStoreForZone", zoneId, false); + } + + @Test + public void testValidateImageStoreForAllZonesWithAllValid() { + DataCenterVO zone1 = Mockito.mock(DataCenterVO.class); + Mockito.when(zone1.getId()).thenReturn(1L); + DataCenterVO zone2 = Mockito.mock(DataCenterVO.class); + Mockito.when(zone2.getId()).thenReturn(2L); + List zones = Arrays.asList(zone1, zone2); + Mockito.when(dataCenterDao.listAllZones()).thenReturn(zones); + + List imageStores = Collections.singletonList(Mockito.mock(ImageStoreVO.class)); + Mockito.when(imageStoreDao.listStoresByZoneId(1L)).thenReturn(imageStores); + Mockito.when(imageStoreDao.listStoresByZoneId(2L)).thenReturn(imageStores); + + ReflectionTestUtils.invokeMethod(kubernetesVersionManager, "validateImageStoreForZone", (Long) null, false); + } + + @Test(expected = InvalidParameterValueException.class) + public void testValidateImageStoreForAllZonesWithSomeMissingStorage() { + DataCenterVO zone1 = Mockito.mock(DataCenterVO.class); + Mockito.when(zone1.getId()).thenReturn(1L); + DataCenterVO zone2 = Mockito.mock(DataCenterVO.class); + Mockito.when(zone2.getId()).thenReturn(2L); + Mockito.when(zone2.getName()).thenReturn("zone-without-storage"); + List zones = Arrays.asList(zone1, zone2); + Mockito.when(dataCenterDao.listAllZones()).thenReturn(zones); + + List imageStores = Collections.singletonList(Mockito.mock(ImageStoreVO.class)); + Mockito.when(imageStoreDao.listStoresByZoneId(1L)).thenReturn(imageStores); + Mockito.when(imageStoreDao.listStoresByZoneId(2L)).thenReturn(Collections.emptyList()); + + ReflectionTestUtils.invokeMethod(kubernetesVersionManager, "validateImageStoreForZone", (Long) null, false); + } } diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/version/KubernetesVersionServiceTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/version/KubernetesVersionServiceTest.java index 455df6b57d4..7ba35169b9e 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/version/KubernetesVersionServiceTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/version/KubernetesVersionServiceTest.java @@ -17,6 +17,9 @@ package com.cloud.kubernetes.version; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertNotNull; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.when; import java.lang.reflect.Field; @@ -25,6 +28,11 @@ import java.util.List; import java.util.UUID; import com.cloud.cpu.CPU; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; +import com.cloud.user.AccountVO; +import com.cloud.user.User; +import com.cloud.user.UserVO; import org.apache.cloudstack.api.command.admin.kubernetes.version.AddKubernetesSupportedVersionCmd; import org.apache.cloudstack.api.command.admin.kubernetes.version.DeleteKubernetesSupportedVersionCmd; import org.apache.cloudstack.api.command.admin.kubernetes.version.UpdateKubernetesSupportedVersionCmd; @@ -63,11 +71,6 @@ import com.cloud.storage.VMTemplateVO; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.template.TemplateApiService; import com.cloud.template.VirtualMachineTemplate; -import com.cloud.user.Account; -import com.cloud.user.AccountManager; -import com.cloud.user.AccountVO; -import com.cloud.user.User; -import com.cloud.user.UserVO; import com.cloud.utils.Pair; import com.cloud.utils.component.ComponentContext; import com.cloud.utils.db.Filter; @@ -75,6 +78,9 @@ import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; +import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; + @RunWith(MockitoJUnitRunner.class) public class KubernetesVersionServiceTest { @@ -94,7 +100,11 @@ public class KubernetesVersionServiceTest { @Mock private DataCenterDao dataCenterDao; @Mock + private ImageStoreDao imageStoreDao; + @Mock private TemplateApiService templateService; + @Mock + private Account accountMock; AutoCloseable closeable; @@ -123,7 +133,12 @@ public class KubernetesVersionServiceTest { DataCenterVO zone = Mockito.mock(DataCenterVO.class); when(dataCenterDao.findById(Mockito.anyLong())).thenReturn(zone); + List imageStores = new ArrayList<>(); + imageStores.add(Mockito.mock(ImageStoreVO.class)); + when(imageStoreDao.listStoresByZoneId(Mockito.anyLong())).thenReturn(imageStores); + TemplateJoinVO templateJoinVO = Mockito.mock(TemplateJoinVO.class); + when(templateJoinVO.getUrl()).thenReturn("https://download.cloudstack.com"); when(templateJoinVO.getState()).thenReturn(ObjectInDataStoreStateMachine.State.Ready); when(templateJoinVO.getArch()).thenReturn(CPU.CPUArch.getDefault()); when(templateJoinDao.findById(Mockito.anyLong())).thenReturn(templateJoinVO); @@ -140,19 +155,66 @@ public class KubernetesVersionServiceTest { @Test public void listKubernetesSupportedVersionsTest() { - ListKubernetesSupportedVersionsCmd cmd = Mockito.mock(ListKubernetesSupportedVersionsCmd.class); - List versionVOs = new ArrayList<>(); - KubernetesSupportedVersionVO versionVO = Mockito.mock(KubernetesSupportedVersionVO.class); - when(versionVO.getSemanticVersion()).thenReturn(KubernetesVersionService.MIN_KUBERNETES_VERSION); - versionVOs.add(versionVO); - when(kubernetesSupportedVersionDao.findById(Mockito.anyLong())).thenReturn(versionVO); - when(kubernetesSupportedVersionDao.searchAndCount(Mockito.any(SearchCriteria.class), - Mockito.any(Filter.class))).thenReturn(new Pair<>(versionVOs, versionVOs.size())); - ListResponse versionsResponse = - kubernetesVersionService.listKubernetesSupportedVersions(cmd); - Assert.assertEquals(versionVOs.size(), versionsResponse.getCount().intValue()); - Assert.assertTrue(CollectionUtils.isNotEmpty(versionsResponse.getResponses())); - Assert.assertEquals(versionVOs.size(), versionsResponse.getResponses().size()); + CallContext callContextMock = Mockito.mock(CallContext.class); + try (MockedStatic callContextMockedStatic = Mockito.mockStatic(CallContext.class)) { + callContextMockedStatic.when(CallContext::current).thenReturn(callContextMock); + final SearchCriteria versionSearchCriteria = Mockito.mock(SearchCriteria.class); + when(callContextMock.getCallingAccount()).thenReturn(accountMock); + ListKubernetesSupportedVersionsCmd cmd = Mockito.mock(ListKubernetesSupportedVersionsCmd.class); + List versionVOs = new ArrayList<>(); + KubernetesSupportedVersionVO versionVO = Mockito.mock(KubernetesSupportedVersionVO.class); + when(versionVO.getSemanticVersion()).thenReturn(KubernetesVersionService.MIN_KUBERNETES_VERSION); + versionVOs.add(versionVO); + when(kubernetesSupportedVersionDao.findById(Mockito.anyLong())).thenReturn(versionVO); + when(kubernetesSupportedVersionDao.searchAndCount(Mockito.any(), Mockito.any(Filter.class))) + .thenReturn(new Pair<>(versionVOs, versionVOs.size())); + ListResponse versionsResponse = + kubernetesVersionService.listKubernetesSupportedVersions(cmd); + Assert.assertEquals(versionVOs.size(), versionsResponse.getCount().intValue()); + Assert.assertTrue(CollectionUtils.isNotEmpty(versionsResponse.getResponses())); + Assert.assertEquals(versionVOs.size(), versionsResponse.getResponses().size()); + } + } + + @Test + public void listKubernetesSupportedVersionsTestWhenAdmin() { + CallContext callContextMock = Mockito.mock(CallContext.class); + try (MockedStatic callContextMockedStatic = Mockito.mockStatic(CallContext.class)) { + callContextMockedStatic.when(CallContext::current).thenReturn(callContextMock); + ListKubernetesSupportedVersionsCmd cmd = Mockito.mock(ListKubernetesSupportedVersionsCmd.class); + List versionVOs = new ArrayList<>(); + KubernetesSupportedVersionVO versionVO = Mockito.mock(KubernetesSupportedVersionVO.class); + when(versionVO.getSemanticVersion()).thenReturn(KubernetesVersionService.MIN_KUBERNETES_VERSION); + versionVOs.add(versionVO); + when(callContextMock.getCallingAccount()).thenReturn(accountMock); + when(kubernetesSupportedVersionDao.findById(Mockito.anyLong())).thenReturn(versionVO); + when(kubernetesSupportedVersionDao.searchAndCount(Mockito.any(), Mockito.any(Filter.class))) + .thenReturn(new Pair<>(versionVOs, versionVOs.size())); + when(accountManager.isRootAdmin(anyLong())).thenReturn(true); + ListResponse response = kubernetesVersionService.listKubernetesSupportedVersions(cmd); + assertNotNull(response.getResponses().get(0).getIsoUrl()); + } + } + + @Test + public void listKubernetesSupportedVersionsTestWhenOtherUser() { + CallContext callContextMock = Mockito.mock(CallContext.class); + try (MockedStatic callContextMockedStatic = Mockito.mockStatic(CallContext.class)) { + callContextMockedStatic.when(CallContext::current).thenReturn(callContextMock); + ListKubernetesSupportedVersionsCmd cmd = Mockito.mock(ListKubernetesSupportedVersionsCmd.class); + List versionVOs = new ArrayList<>(); + KubernetesSupportedVersionVO versionVO = Mockito.mock(KubernetesSupportedVersionVO.class); + when(versionVO.getSemanticVersion()).thenReturn(KubernetesVersionService.MIN_KUBERNETES_VERSION); + versionVOs.add(versionVO); + when(callContextMock.getCallingAccount()).thenReturn(accountMock); + when(kubernetesSupportedVersionDao.findById(Mockito.anyLong())).thenReturn(versionVO); + when(kubernetesSupportedVersionDao.searchAndCount(Mockito.any(), Mockito.any(Filter.class))) + .thenReturn(new Pair<>(versionVOs, versionVOs.size())); + when(accountManager.isRootAdmin(anyLong())).thenReturn(false); + when(accountMock.getId()).thenReturn(2L); + ListResponse response = kubernetesVersionService.listKubernetesSupportedVersions(cmd); + assertNull(response.getResponses().get(0).getIsoUrl()); + } } @Test(expected = InvalidParameterValueException.class) @@ -224,7 +286,6 @@ public class KubernetesVersionServiceTest { mockedComponentContext.when(() -> ComponentContext.inject(Mockito.any(RegisterIsoCmd.class))).thenReturn( new RegisterIsoCmd()); mockedCallContext.when(CallContext::current).thenReturn(callContext); - when(templateService.registerIso(Mockito.any(RegisterIsoCmd.class))).thenReturn( Mockito.mock(VirtualMachineTemplate.class)); VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class); diff --git a/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/driver/DateraPrimaryDataStoreDriver.java b/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/driver/DateraPrimaryDataStoreDriver.java index dcf84525748..62393610499 100644 --- a/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/driver/DateraPrimaryDataStoreDriver.java +++ b/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/driver/DateraPrimaryDataStoreDriver.java @@ -563,7 +563,7 @@ public class DateraPrimaryDataStoreDriver implements PrimaryDataStoreDriver { private long getUsedBytes(StoragePool storagePool, long volumeIdToIgnore) { long usedSpaceBytes = 0; - List lstVolumes = _volumeDao.findByPoolId(storagePool.getId(), null); + List lstVolumes = _volumeDao.findNonDestroyedVolumesByPoolId(storagePool.getId(), null); if (lstVolumes != null) { for (VolumeVO volume : lstVolumes) { diff --git a/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/provider/DateraHostListener.java b/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/provider/DateraHostListener.java index a0dc23da486..08bc89737f2 100644 --- a/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/provider/DateraHostListener.java +++ b/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/provider/DateraHostListener.java @@ -247,7 +247,7 @@ public class DateraHostListener implements HypervisorHostListener { List storagePaths = new ArrayList<>(); // If you do not pass in null for the second parameter, you only get back applicable ROOT disks. - List volumes = _volumeDao.findByPoolId(storagePoolId, null); + List volumes = _volumeDao.findNonDestroyedVolumesByPoolId(storagePoolId, null); if (volumes != null) { for (VolumeVO volume : volumes) { @@ -317,7 +317,7 @@ public class DateraHostListener implements HypervisorHostListener { StoragePoolVO storagePool = _storagePoolDao.findById(storagePoolId); // If you do not pass in null for the second parameter, you only get back applicable ROOT disks. - List volumes = _volumeDao.findByPoolId(storagePoolId, null); + List volumes = _volumeDao.findNonDestroyedVolumesByPoolId(storagePoolId, null); if (volumes != null) { for (VolumeVO volume : volumes) { diff --git a/plugins/storage/volume/linstor/CHANGELOG.md b/plugins/storage/volume/linstor/CHANGELOG.md index 47d1ddeb06c..a5d609325d6 100644 --- a/plugins/storage/volume/linstor/CHANGELOG.md +++ b/plugins/storage/volume/linstor/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to Linstor CloudStack plugin will be documented in this file The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [2026-01-17] + +### Added + +- Support live migrate from other primary storage + ## [2025-12-18] ### Changed diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java index c2bce6e5a04..27b5f7ef7ec 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java @@ -21,33 +21,25 @@ import com.linbit.linstor.api.CloneWaiter; import com.linbit.linstor.api.DevelopersApi; import com.linbit.linstor.api.model.ApiCallRc; import com.linbit.linstor.api.model.ApiCallRcList; -import com.linbit.linstor.api.model.AutoSelectFilter; import com.linbit.linstor.api.model.LayerType; -import com.linbit.linstor.api.model.Properties; import com.linbit.linstor.api.model.ResourceDefinition; import com.linbit.linstor.api.model.ResourceDefinitionCloneRequest; import com.linbit.linstor.api.model.ResourceDefinitionCloneStarted; import com.linbit.linstor.api.model.ResourceDefinitionCreate; import com.linbit.linstor.api.model.ResourceDefinitionModify; -import com.linbit.linstor.api.model.ResourceGroup; -import com.linbit.linstor.api.model.ResourceGroupSpawn; import com.linbit.linstor.api.model.ResourceMakeAvailable; import com.linbit.linstor.api.model.ResourceWithVolumes; import com.linbit.linstor.api.model.Snapshot; import com.linbit.linstor.api.model.SnapshotRestore; -import com.linbit.linstor.api.model.VolumeDefinition; import com.linbit.linstor.api.model.VolumeDefinitionModify; import javax.annotation.Nonnull; -import javax.annotation.Nullable; import javax.inject.Inject; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.stream.Collectors; @@ -117,10 +109,9 @@ import org.apache.cloudstack.storage.snapshot.SnapshotObject; import org.apache.cloudstack.storage.to.SnapshotObjectTO; import org.apache.cloudstack.storage.to.VolumeObjectTO; import org.apache.cloudstack.storage.volume.VolumeObject; - -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.LogManager; import org.apache.commons.collections.CollectionUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.nio.charset.StandardCharsets; @@ -335,275 +326,11 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver } } - private void logLinstorAnswer(@Nonnull ApiCallRc answer) { - if (answer.isError()) { - logger.error(answer.getMessage()); - } else if (answer.isWarning()) { - logger.warn(answer.getMessage()); - } else if (answer.isInfo()) { - logger.info(answer.getMessage()); - } - } - - private void logLinstorAnswers(@Nonnull ApiCallRcList answers) { - answers.forEach(this::logLinstorAnswer); - } - - private void checkLinstorAnswersThrow(@Nonnull ApiCallRcList answers) { - logLinstorAnswers(answers); - if (answers.hasError()) - { - String errMsg = answers.stream() - .filter(ApiCallRc::isError) - .findFirst() - .map(ApiCallRc::getMessage).orElse("Unknown linstor error"); - throw new CloudRuntimeException(errMsg); - } - } - private String checkLinstorAnswers(@Nonnull ApiCallRcList answers) { - logLinstorAnswers(answers); + LinstorUtil.logLinstorAnswers(answers); return answers.stream().filter(ApiCallRc::isError).findFirst().map(ApiCallRc::getMessage).orElse(null); } - private void applyQoSSettings(StoragePoolVO storagePool, DevelopersApi api, String rscName, Long maxIops) - throws ApiException - { - Long currentQosIops = null; - List vlmDfns = api.volumeDefinitionList(rscName, null, null); - if (!vlmDfns.isEmpty()) - { - Properties props = vlmDfns.get(0).getProps(); - long iops = Long.parseLong(props.getOrDefault("sys/fs/blkio_throttle_write_iops", "0")); - currentQosIops = iops > 0 ? iops : null; - } - - if (!Objects.equals(maxIops, currentQosIops)) - { - VolumeDefinitionModify vdm = new VolumeDefinitionModify(); - if (maxIops != null) - { - Properties props = new Properties(); - props.put("sys/fs/blkio_throttle_read_iops", "" + maxIops); - props.put("sys/fs/blkio_throttle_write_iops", "" + maxIops); - vdm.overrideProps(props); - logger.info("Apply qos setting: " + maxIops + " to " + rscName); - } - else - { - logger.info("Remove QoS setting for " + rscName); - vdm.deleteProps(Arrays.asList("sys/fs/blkio_throttle_read_iops", "sys/fs/blkio_throttle_write_iops")); - } - ApiCallRcList answers = api.volumeDefinitionModify(rscName, 0, vdm); - checkLinstorAnswersThrow(answers); - - Long capacityIops = storagePool.getCapacityIops(); - if (capacityIops != null) - { - long vcIops = currentQosIops != null ? currentQosIops * -1 : 0; - long vMaxIops = maxIops != null ? maxIops : 0; - long newIops = vcIops + vMaxIops; - capacityIops -= newIops; - logger.info(String.format("Current storagepool %s iops capacity: %d", storagePool, capacityIops)); - storagePool.setCapacityIops(Math.max(0, capacityIops)); - _storagePoolDao.update(storagePool.getId(), storagePool); - } - } - } - - private String getRscGrp(StoragePool storagePool) { - return storagePool.getUserInfo() != null && !storagePool.getUserInfo().isEmpty() ? - storagePool.getUserInfo() : "DfltRscGrp"; - } - - /** - * Returns the layerlist of the resourceGroup with encryption(LUKS) added above STORAGE. - * If the resourceGroup layer list already contains LUKS this layer list will be returned. - * @param api Linstor developers API - * @param resourceGroup Resource group to get the encryption layer list - * @return layer list with LUKS added - */ - public List getEncryptedLayerList(DevelopersApi api, String resourceGroup) { - try { - List rscGrps = api.resourceGroupList( - Collections.singletonList(resourceGroup), Collections.emptyList(), null, null); - - if (CollectionUtils.isEmpty(rscGrps)) { - throw new CloudRuntimeException( - String.format("Resource Group %s not found on Linstor cluster.", resourceGroup)); - } - - final ResourceGroup rscGrp = rscGrps.get(0); - List layers = Arrays.asList(LayerType.DRBD, LayerType.LUKS, LayerType.STORAGE); - List curLayerStack = rscGrp.getSelectFilter() != null ? - rscGrp.getSelectFilter().getLayerStack() : Collections.emptyList(); - if (CollectionUtils.isNotEmpty(curLayerStack)) { - layers = curLayerStack.stream().map(LayerType::valueOf).collect(Collectors.toList()); - if (!layers.contains(LayerType.LUKS)) { - layers.add(layers.size() - 1, LayerType.LUKS); // lowest layer is STORAGE - } - } - return layers; - } catch (ApiException e) { - throw new CloudRuntimeException( - String.format("Resource Group %s not found on Linstor cluster.", resourceGroup)); - } - } - - /** - * Spawns a new Linstor resource with the given arguments. - * @param api - * @param newRscName - * @param sizeInBytes - * @param isTemplate - * @param rscGrpName - * @param volName - * @param vmName - * @throws ApiException - */ - private void spawnResource( - DevelopersApi api, String newRscName, long sizeInBytes, boolean isTemplate, String rscGrpName, - String volName, String vmName, @Nullable Long passPhraseId, @Nullable byte[] passPhrase) throws ApiException - { - ResourceGroupSpawn rscGrpSpawn = new ResourceGroupSpawn(); - rscGrpSpawn.setResourceDefinitionName(newRscName); - rscGrpSpawn.addVolumeSizesItem(sizeInBytes / 1024); - if (passPhraseId != null) { - AutoSelectFilter asf = new AutoSelectFilter(); - List luksLayers = getEncryptedLayerList(api, rscGrpName); - asf.setLayerStack(luksLayers.stream().map(LayerType::toString).collect(Collectors.toList())); - rscGrpSpawn.setSelectFilter(asf); - if (passPhrase != null) { - String utf8Passphrase = new String(passPhrase, StandardCharsets.UTF_8); - rscGrpSpawn.setVolumePassphrases(Collections.singletonList(utf8Passphrase)); - } - } - - if (isTemplate) { - Properties props = new Properties(); - props.put(LinstorUtil.getTemplateForAuxPropKey(rscGrpName), "true"); - rscGrpSpawn.setResourceDefinitionProps(props); - } - - logger.info("Linstor: Spawn resource " + newRscName); - ApiCallRcList answers = api.resourceGroupSpawn(rscGrpName, rscGrpSpawn); - checkLinstorAnswersThrow(answers); - - answers = LinstorUtil.applyAuxProps(api, newRscName, volName, vmName); - checkLinstorAnswersThrow(answers); - } - - /** - * Condition if a template resource can be shared with the given resource group. - * @param tgtRscGrp - * @param tgtLayerStack - * @param rg - * @return True if the template resource can be shared, else false. - */ - private boolean canShareTemplateForResourceGroup( - ResourceGroup tgtRscGrp, List tgtLayerStack, ResourceGroup rg) { - List rgLayerStack = rg.getSelectFilter() != null ? - rg.getSelectFilter().getLayerStack() : null; - return Objects.equals(tgtLayerStack, rgLayerStack) && - Objects.equals(tgtRscGrp.getSelectFilter().getStoragePoolList(), - rg.getSelectFilter().getStoragePoolList()); - } - - /** - * Searches for a shareable template for this rscGrpName and sets the aux template property. - * @param api - * @param rscName - * @param rscGrpName - * @param existingRDs - * @return - * @throws ApiException - */ - private boolean foundShareableTemplate( - DevelopersApi api, String rscName, String rscGrpName, - List> existingRDs) throws ApiException { - if (!existingRDs.isEmpty()) { - ResourceGroup tgtRscGrp = api.resourceGroupList( - Collections.singletonList(rscGrpName), null, null, null).get(0); - List tgtLayerStack = tgtRscGrp.getSelectFilter() != null ? - tgtRscGrp.getSelectFilter().getLayerStack() : null; - - // check if there is already a template copy, that we could reuse - // this means if select filters are similar enough to allow cloning from - for (Pair rdPair : existingRDs) { - ResourceGroup rg = rdPair.second(); - if (canShareTemplateForResourceGroup(tgtRscGrp, tgtLayerStack, rg)) { - LinstorUtil.setAuxTemplateForProperty(api, rscName, rscGrpName); - return true; - } - } - } - return false; - } - - /** - * Creates a new Linstor resource. - * @param rscName - * @param sizeInBytes - * @param volName - * @param vmName - * @param api - * @param rscGrp - * @param poolId - * @param isTemplate indicates if the resource is a template - * @return true if a new resource was created, false if it already existed or was reused. - */ - private boolean createResourceBase( - String rscName, long sizeInBytes, String volName, String vmName, - @Nullable Long passPhraseId, @Nullable byte[] passPhrase, DevelopersApi api, - String rscGrp, long poolId, boolean isTemplate) - { - try - { - logger.debug("createRscBase: {} :: {} :: {}", rscName, rscGrp, isTemplate); - List> existingRDs = LinstorUtil.getRDAndRGListStartingWith(api, rscName); - - String fullRscName = String.format("%s-%d", rscName, poolId); - boolean alreadyCreated = existingRDs.stream() - .anyMatch(p -> p.first().getName().equalsIgnoreCase(fullRscName)) || - existingRDs.stream().anyMatch(p -> p.first().getProps().containsKey(LinstorUtil.getTemplateForAuxPropKey(rscGrp))); - if (!alreadyCreated) { - boolean createNewRsc = !foundShareableTemplate(api, rscName, rscGrp, existingRDs); - if (createNewRsc) { - String newRscName = existingRDs.isEmpty() ? rscName : fullRscName; - spawnResource(api, newRscName, sizeInBytes, isTemplate, rscGrp, - volName, vmName, passPhraseId, passPhrase); - } - return createNewRsc; - } - return false; - } catch (ApiException apiEx) - { - logger.error("Linstor: ApiEx - " + apiEx.getMessage()); - throw new CloudRuntimeException(apiEx.getBestMessage(), apiEx); - } - } - - private String createResource(VolumeInfo vol, StoragePoolVO storagePoolVO) { - DevelopersApi linstorApi = LinstorUtil.getLinstorAPI(storagePoolVO.getHostAddress()); - final String rscGrp = getRscGrp(storagePoolVO); - - final String rscName = LinstorUtil.RSC_PREFIX + vol.getUuid(); - createResourceBase( - rscName, vol.getSize(), vol.getName(), vol.getAttachedVmName(), vol.getPassphraseId(), vol.getPassphrase(), - linstorApi, rscGrp, storagePoolVO.getId(), false); - - try - { - applyQoSSettings(storagePoolVO, linstorApi, rscName, vol.getMaxIops()); - - return LinstorUtil.getDevicePath(linstorApi, rscName); - } catch (ApiException apiEx) - { - logger.error("Linstor: ApiEx - " + apiEx.getMessage()); - throw new CloudRuntimeException(apiEx.getBestMessage(), apiEx); - } - } - private void resizeResource(DevelopersApi api, String resourceName, long sizeByte) throws ApiException { VolumeDefinitionModify dfm = new VolumeDefinitionModify(); dfm.setSizeKib(sizeByte / 1024); @@ -688,13 +415,14 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver try { ResourceDefinition templateRD = LinstorUtil.findResourceDefinition( - linstorApi, templateRscName, getRscGrp(storagePoolVO)); + linstorApi, templateRscName, LinstorUtil.getRscGrp(storagePoolVO)); final String cloneRes = templateRD != null ? templateRD.getName() : templateRscName; logger.info("Clone resource definition {} to {}", cloneRes, rscName); ResourceDefinitionCloneRequest cloneRequest = new ResourceDefinitionCloneRequest(); cloneRequest.setName(rscName); if (volumeInfo.getPassphraseId() != null) { - List encryptionLayer = getEncryptedLayerList(linstorApi, getRscGrp(storagePoolVO)); + List encryptionLayer = LinstorUtil.getEncryptedLayerList( + linstorApi, LinstorUtil.getRscGrp(storagePoolVO)); cloneRequest.setLayerList(encryptionLayer); if (volumeInfo.getPassphrase() != null) { String utf8Passphrase = new String(volumeInfo.getPassphrase(), StandardCharsets.UTF_8); @@ -704,7 +432,7 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver ResourceDefinitionCloneStarted cloneStarted = linstorApi.resourceDefinitionClone( cloneRes, cloneRequest); - checkLinstorAnswersThrow(cloneStarted.getMessages()); + LinstorUtil.checkLinstorAnswersThrow(cloneStarted.getMessages()); if (!CloneWaiter.waitFor(linstorApi, cloneStarted)) { throw new CloudRuntimeException("Clone for resource " + rscName + " failed."); @@ -716,11 +444,12 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver resizeResource(linstorApi, rscName, volumeInfo.getSize()); } - updateRscGrpIfNecessary(linstorApi, rscName, getRscGrp(storagePoolVO)); + updateRscGrpIfNecessary(linstorApi, rscName, LinstorUtil.getRscGrp(storagePoolVO)); deleteTemplateForProps(linstorApi, rscName); LinstorUtil.applyAuxProps(linstorApi, rscName, volumeInfo.getName(), volumeInfo.getAttachedVmName()); - applyQoSSettings(storagePoolVO, linstorApi, rscName, volumeInfo.getMaxIops()); + LinstorUtil.applyQoSSettings( + _storagePoolDao, storagePoolVO, linstorApi, rscName, volumeInfo.getMaxIops()); return LinstorUtil.getDevicePath(linstorApi, rscName); } catch (ApiException apiEx) { @@ -744,7 +473,7 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver } private String createResourceFromSnapshot(long csSnapshotId, String rscName, StoragePoolVO storagePoolVO) { - final String rscGrp = getRscGrp(storagePoolVO); + final String rscGrp = LinstorUtil.getRscGrp(storagePoolVO); final DevelopersApi linstorApi = LinstorUtil.getLinstorAPI(storagePoolVO.getHostAddress()); SnapshotVO snapshotVO = _snapshotDao.findById(csSnapshotId); @@ -757,22 +486,22 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver logger.debug("Create new resource definition: " + rscName); ResourceDefinitionCreate rdCreate = createResourceDefinitionCreate(rscName, rscGrp); ApiCallRcList answers = linstorApi.resourceDefinitionCreate(rdCreate); - checkLinstorAnswersThrow(answers); + LinstorUtil.checkLinstorAnswersThrow(answers); SnapshotRestore snapshotRestore = new SnapshotRestore(); snapshotRestore.toResource(rscName); logger.debug("Create new volume definition for snapshot: " + cloneRes + ":" + snapName); answers = linstorApi.resourceSnapshotsRestoreVolumeDefinition(cloneRes, snapName, snapshotRestore); - checkLinstorAnswersThrow(answers); + LinstorUtil.checkLinstorAnswersThrow(answers); // restore snapshot to new resource logger.info("Restore resource from snapshot: " + cloneRes + ":" + snapName); answers = linstorApi.resourceSnapshotRestore(cloneRes, snapName, snapshotRestore); - checkLinstorAnswersThrow(answers); + LinstorUtil.checkLinstorAnswersThrow(answers); LinstorUtil.applyAuxProps(linstorApi, rscName, volumeVO.getName(), null); - applyQoSSettings(storagePoolVO, linstorApi, rscName, volumeVO.getMaxIops()); + LinstorUtil.applyQoSSettings(_storagePoolDao, storagePoolVO, linstorApi, rscName, volumeVO.getMaxIops()); return LinstorUtil.getDevicePath(linstorApi, rscName); } catch (ApiException apiEx) { @@ -790,7 +519,7 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver } else if (csTemplateId > 0) { return cloneResource(csTemplateId, volumeInfo, storagePoolVO); } else { - return createResource(volumeInfo, storagePoolVO); + return LinstorUtil.createResource(volumeInfo, storagePoolVO, _storagePoolDao); } } @@ -1140,7 +869,7 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver String rscName, String snapshotName, String restoredName) throws ApiException { - final String rscGrp = getRscGrp(storagePoolVO); + final String rscGrp = LinstorUtil.getRscGrp(storagePoolVO); // try to delete -rst resource, could happen if the copy failed and noone deleted it. deleteResourceDefinition(storagePoolVO, restoredName); ResourceDefinitionCreate rdc = createResourceDefinitionCreate(restoredName, rscGrp); @@ -1185,7 +914,7 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver final StoragePoolVO pool = _storagePoolDao.findById(dstData.getDataStore().getId()); final DevelopersApi api = LinstorUtil.getLinstorAPI(pool.getHostAddress()); final String rscName = LinstorUtil.RSC_PREFIX + dstData.getUuid(); - boolean newCreated = createResourceBase( + boolean newCreated = LinstorUtil.createResourceBase( LinstorUtil.RSC_PREFIX + dstData.getUuid(), tInfo.getSize(), tInfo.getName(), @@ -1193,9 +922,10 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver null, null, api, - getRscGrp(pool), + LinstorUtil.getRscGrp(pool), pool.getId(), - true); + true, + false); Answer answer; if (newCreated) { @@ -1429,7 +1159,7 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver { resizeResource(api, rscName, resizeParameter.newSize); - applyQoSSettings(pool, api, rscName, resizeParameter.newMaxIops); + LinstorUtil.applyQoSSettings(_storagePoolDao, pool, api, rscName, resizeParameter.newMaxIops); { final VolumeVO volume = _volumeDao.findById(vol.getId()); volume.setMinIops(resizeParameter.newMinIops); @@ -1534,7 +1264,7 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver @Override public Pair getStorageStats(StoragePool storagePool) { logger.debug(String.format("Requesting storage stats: %s", storagePool)); - return LinstorUtil.getStorageStats(storagePool.getHostAddress(), getRscGrp(storagePool)); + return LinstorUtil.getStorageStats(storagePool.getHostAddress(), LinstorUtil.getRscGrp(storagePool)); } @Override diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java index 4196c12b116..7c45493dddc 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java @@ -22,6 +22,8 @@ import com.linbit.linstor.api.ApiException; import com.linbit.linstor.api.DevelopersApi; import com.linbit.linstor.api.model.ApiCallRc; import com.linbit.linstor.api.model.ApiCallRcList; +import com.linbit.linstor.api.model.AutoSelectFilter; +import com.linbit.linstor.api.model.LayerType; import com.linbit.linstor.api.model.Node; import com.linbit.linstor.api.model.Properties; import com.linbit.linstor.api.model.ProviderKind; @@ -29,24 +31,36 @@ import com.linbit.linstor.api.model.Resource; import com.linbit.linstor.api.model.ResourceDefinition; import com.linbit.linstor.api.model.ResourceDefinitionModify; import com.linbit.linstor.api.model.ResourceGroup; +import com.linbit.linstor.api.model.ResourceGroupSpawn; import com.linbit.linstor.api.model.ResourceWithVolumes; import com.linbit.linstor.api.model.StoragePool; import com.linbit.linstor.api.model.Volume; +import com.linbit.linstor.api.model.VolumeDefinition; +import com.linbit.linstor.api.model.VolumeDefinitionModify; import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.stream.Collectors; import com.cloud.hypervisor.kvm.storage.KVMStoragePool; import com.cloud.utils.Pair; import com.cloud.utils.exception.CloudRuntimeException; -import org.apache.logging.log4j.Logger; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.commons.collections.CollectionUtils; import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.nio.charset.StandardCharsets; public class LinstorUtil { protected static Logger LOGGER = LogManager.getLogger(LinstorUtil.class); @@ -56,6 +70,8 @@ public class LinstorUtil { public static final String RSC_GROUP = "resourceGroup"; public static final String CS_TEMPLATE_FOR_PREFIX = "_cs-template-for-"; + public static final String LIN_PROP_DRBDOPT_EXACT_SIZE = "DrbdOptions/ExactSize"; + public static final String TEMP_VOLUME_ID = "tempVolumeId"; public static final String CLUSTER_DEFAULT_MIN_IOPS = "clusterDefaultMinIops"; @@ -76,6 +92,32 @@ public class LinstorUtil { .orElse((answers.get(0)).getMessage()) : null; } + public static void logLinstorAnswer(@Nonnull ApiCallRc answer) { + if (answer.isError()) { + LOGGER.error(answer.getMessage()); + } else if (answer.isWarning()) { + LOGGER.warn(answer.getMessage()); + } else if (answer.isInfo()) { + LOGGER.info(answer.getMessage()); + } + } + + public static void logLinstorAnswers(@Nonnull ApiCallRcList answers) { + answers.forEach(LinstorUtil::logLinstorAnswer); + } + + public static void checkLinstorAnswersThrow(@Nonnull ApiCallRcList answers) { + logLinstorAnswers(answers); + if (answers.hasError()) + { + String errMsg = answers.stream() + .filter(ApiCallRc::isError) + .findFirst() + .map(ApiCallRc::getMessage).orElse("Unknown linstor error"); + throw new CloudRuntimeException(errMsg); + } + } + public static List getLinstorNodeNames(@Nonnull DevelopersApi api) throws ApiException { List nodes = api.nodeList( @@ -488,4 +530,253 @@ public class LinstorUtil { } return false; } + + public static String getRscGrp(com.cloud.storage.StoragePool storagePool) { + return storagePool.getUserInfo() != null && !storagePool.getUserInfo().isEmpty() ? + storagePool.getUserInfo() : "DfltRscGrp"; + } + + /** + * Condition if a template resource can be shared with the given resource group. + * @param tgtRscGrp + * @param tgtLayerStack + * @param rg + * @return True if the template resource can be shared, else false. + */ + private static boolean canShareTemplateForResourceGroup( + ResourceGroup tgtRscGrp, List tgtLayerStack, ResourceGroup rg) { + List rgLayerStack = rg.getSelectFilter() != null ? + rg.getSelectFilter().getLayerStack() : null; + return Objects.equals(tgtLayerStack, rgLayerStack) && + Objects.equals(tgtRscGrp.getSelectFilter().getStoragePoolList(), + rg.getSelectFilter().getStoragePoolList()); + } + + /** + * Searches for a shareable template for this rscGrpName and sets the aux template property. + * @param api + * @param rscName + * @param rscGrpName + * @param existingRDs + * @return + * @throws ApiException + */ + private static boolean foundShareableTemplate( + DevelopersApi api, String rscName, String rscGrpName, + List> existingRDs) throws ApiException { + if (!existingRDs.isEmpty()) { + ResourceGroup tgtRscGrp = api.resourceGroupList( + Collections.singletonList(rscGrpName), null, null, null).get(0); + List tgtLayerStack = tgtRscGrp.getSelectFilter() != null ? + tgtRscGrp.getSelectFilter().getLayerStack() : null; + + // check if there is already a template copy, that we could reuse + // this means if select filters are similar enough to allow cloning from + for (Pair rdPair : existingRDs) { + ResourceGroup rg = rdPair.second(); + if (canShareTemplateForResourceGroup(tgtRscGrp, tgtLayerStack, rg)) { + LinstorUtil.setAuxTemplateForProperty(api, rscName, rscGrpName); + return true; + } + } + } + return false; + } + + /** + * Returns the layerlist of the resourceGroup with encryption(LUKS) added above STORAGE. + * If the resourceGroup layer list already contains LUKS this layer list will be returned. + * @param api Linstor developers API + * @param resourceGroup Resource group to get the encryption layer list + * @return layer list with LUKS added + */ + public static List getEncryptedLayerList(DevelopersApi api, String resourceGroup) { + try { + List rscGrps = api.resourceGroupList( + Collections.singletonList(resourceGroup), Collections.emptyList(), null, null); + + if (CollectionUtils.isEmpty(rscGrps)) { + throw new CloudRuntimeException( + String.format("Resource Group %s not found on Linstor cluster.", resourceGroup)); + } + + final ResourceGroup rscGrp = rscGrps.get(0); + List layers = Arrays.asList(LayerType.DRBD, LayerType.LUKS, LayerType.STORAGE); + List curLayerStack = rscGrp.getSelectFilter() != null ? + rscGrp.getSelectFilter().getLayerStack() : Collections.emptyList(); + if (CollectionUtils.isNotEmpty(curLayerStack)) { + layers = curLayerStack.stream().map(LayerType::valueOf).collect(Collectors.toList()); + if (!layers.contains(LayerType.LUKS)) { + layers.add(layers.size() - 1, LayerType.LUKS); // lowest layer is STORAGE + } + } + return layers; + } catch (ApiException e) { + throw new CloudRuntimeException( + String.format("Resource Group %s not found on Linstor cluster.", resourceGroup)); + } + } + + /** + * Spawns a new Linstor resource with the given arguments. + * @param api + * @param newRscName + * @param sizeInBytes + * @param isTemplate + * @param rscGrpName + * @param volName + * @param vmName + * @throws ApiException + */ + private static void spawnResource( + DevelopersApi api, String newRscName, long sizeInBytes, boolean isTemplate, String rscGrpName, + String volName, String vmName, @Nullable Long passPhraseId, @Nullable byte[] passPhrase, + boolean exactSize) throws ApiException + { + ResourceGroupSpawn rscGrpSpawn = new ResourceGroupSpawn(); + rscGrpSpawn.setResourceDefinitionName(newRscName); + rscGrpSpawn.addVolumeSizesItem(sizeInBytes / 1024); + if (passPhraseId != null) { + AutoSelectFilter asf = new AutoSelectFilter(); + List luksLayers = getEncryptedLayerList(api, rscGrpName); + asf.setLayerStack(luksLayers.stream().map(LayerType::toString).collect(Collectors.toList())); + rscGrpSpawn.setSelectFilter(asf); + if (passPhrase != null) { + String utf8Passphrase = new String(passPhrase, StandardCharsets.UTF_8); + rscGrpSpawn.setVolumePassphrases(Collections.singletonList(utf8Passphrase)); + } + } + + Properties props = new Properties(); + if (isTemplate) { + props.put(LinstorUtil.getTemplateForAuxPropKey(rscGrpName), "true"); + } + if (exactSize) { + props.put(LIN_PROP_DRBDOPT_EXACT_SIZE, "true"); + } + rscGrpSpawn.setResourceDefinitionProps(props); + + LOGGER.info("Linstor: Spawn resource " + newRscName); + ApiCallRcList answers = api.resourceGroupSpawn(rscGrpName, rscGrpSpawn); + checkLinstorAnswersThrow(answers); + + answers = LinstorUtil.applyAuxProps(api, newRscName, volName, vmName); + checkLinstorAnswersThrow(answers); + } + + /** + * Creates a new Linstor resource. + * @param rscName + * @param sizeInBytes + * @param volName + * @param vmName + * @param api + * @param rscGrp + * @param poolId + * @param isTemplate indicates if the resource is a template + * @return true if a new resource was created, false if it already existed or was reused. + */ + public static boolean createResourceBase( + String rscName, long sizeInBytes, String volName, String vmName, + @Nullable Long passPhraseId, @Nullable byte[] passPhrase, DevelopersApi api, + String rscGrp, long poolId, boolean isTemplate, boolean exactSize) + { + try + { + LOGGER.debug("createRscBase: {} :: {} :: {} :: {}", rscName, rscGrp, isTemplate, exactSize); + List> existingRDs = LinstorUtil.getRDAndRGListStartingWith(api, rscName); + + String fullRscName = String.format("%s-%d", rscName, poolId); + boolean alreadyCreated = existingRDs.stream() + .anyMatch(p -> p.first().getName().equalsIgnoreCase(fullRscName)) || + existingRDs.stream().anyMatch(p -> p.first().getProps().containsKey(LinstorUtil.getTemplateForAuxPropKey(rscGrp))); + if (!alreadyCreated) { + boolean createNewRsc = !foundShareableTemplate(api, rscName, rscGrp, existingRDs); + if (createNewRsc) { + String newRscName = existingRDs.isEmpty() ? rscName : fullRscName; + spawnResource(api, newRscName, sizeInBytes, isTemplate, rscGrp, + volName, vmName, passPhraseId, passPhrase, exactSize); + } + return createNewRsc; + } + return false; + } catch (ApiException apiEx) + { + LOGGER.error("Linstor: ApiEx - {}", apiEx.getMessage()); + throw new CloudRuntimeException(apiEx.getBestMessage(), apiEx); + } + } + + public static void applyQoSSettings(PrimaryDataStoreDao primaryDataStoreDao, + StoragePoolVO storagePool, DevelopersApi api, String rscName, Long maxIops) + throws ApiException + { + Long currentQosIops = null; + List vlmDfns = api.volumeDefinitionList(rscName, null, null); + if (!vlmDfns.isEmpty()) + { + Properties props = vlmDfns.get(0).getProps(); + long iops = Long.parseLong(props.getOrDefault("sys/fs/blkio_throttle_write_iops", "0")); + currentQosIops = iops > 0 ? iops : null; + } + + if (!Objects.equals(maxIops, currentQosIops)) + { + VolumeDefinitionModify vdm = new VolumeDefinitionModify(); + if (maxIops != null) + { + Properties props = new Properties(); + props.put("sys/fs/blkio_throttle_read_iops", "" + maxIops); + props.put("sys/fs/blkio_throttle_write_iops", "" + maxIops); + vdm.overrideProps(props); + LOGGER.info("Apply qos setting: {} to {}", maxIops, rscName); + } + else + { + LOGGER.info("Remove QoS setting for {}", rscName); + vdm.deleteProps(Arrays.asList("sys/fs/blkio_throttle_read_iops", "sys/fs/blkio_throttle_write_iops")); + } + ApiCallRcList answers = api.volumeDefinitionModify(rscName, 0, vdm); + LinstorUtil.checkLinstorAnswersThrow(answers); + + Long capacityIops = storagePool.getCapacityIops(); + if (capacityIops != null) + { + long vcIops = currentQosIops != null ? currentQosIops * -1 : 0; + long vMaxIops = maxIops != null ? maxIops : 0; + long newIops = vcIops + vMaxIops; + capacityIops -= newIops; + LOGGER.info("Current storagepool {} iops capacity: {}", storagePool, capacityIops); + storagePool.setCapacityIops(Math.max(0, capacityIops)); + primaryDataStoreDao.update(storagePool.getId(), storagePool); + } + } + } + + public static String createResource(VolumeInfo vol, StoragePoolVO storagePoolVO, + PrimaryDataStoreDao primaryDataStoreDao) { + return createResource(vol, storagePoolVO, primaryDataStoreDao, false); + } + + public static String createResource(VolumeInfo vol, StoragePoolVO storagePoolVO, + PrimaryDataStoreDao primaryDataStoreDao, boolean exactSize) { + DevelopersApi linstorApi = LinstorUtil.getLinstorAPI(storagePoolVO.getHostAddress()); + final String rscGrp = getRscGrp(storagePoolVO); + + final String rscName = LinstorUtil.RSC_PREFIX + vol.getUuid(); + createResourceBase( + rscName, vol.getSize(), vol.getName(), vol.getAttachedVmName(), vol.getPassphraseId(), vol.getPassphrase(), + linstorApi, rscGrp, storagePoolVO.getId(), false, exactSize); + + try + { + applyQoSSettings(primaryDataStoreDao, storagePoolVO, linstorApi, rscName, vol.getMaxIops()); + + return LinstorUtil.getDevicePath(linstorApi, rscName); + } catch (ApiException apiEx) + { + LOGGER.error("Linstor: ApiEx - " + apiEx.getMessage()); + throw new CloudRuntimeException(apiEx.getBestMessage(), apiEx); + } + } } diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/motion/LinstorDataMotionStrategy.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/motion/LinstorDataMotionStrategy.java new file mode 100644 index 00000000000..cab2820f09a --- /dev/null +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/motion/LinstorDataMotionStrategy.java @@ -0,0 +1,437 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.cloudstack.storage.motion; + +import com.linbit.linstor.api.ApiException; +import com.linbit.linstor.api.DevelopersApi; +import com.linbit.linstor.api.model.ApiCallRcList; +import com.linbit.linstor.api.model.ResourceDefinition; +import com.linbit.linstor.api.model.ResourceDefinitionModify; + +import javax.inject.Inject; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.cloud.agent.AgentManager; +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.MigrateAnswer; +import com.cloud.agent.api.MigrateCommand; +import com.cloud.agent.api.PrepareForMigrationCommand; +import com.cloud.agent.api.to.DataObjectType; +import com.cloud.agent.api.to.VirtualMachineTO; +import com.cloud.exception.AgentUnavailableException; +import com.cloud.exception.OperationTimedoutException; +import com.cloud.host.Host; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.storage.Storage; +import com.cloud.storage.StorageManager; +import com.cloud.storage.Volume; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.GuestOSCategoryDao; +import com.cloud.storage.dao.GuestOSDao; +import com.cloud.storage.dao.SnapshotDao; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.dao.VMInstanceDao; +import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; +import org.apache.cloudstack.engine.subsystem.api.storage.DataMotionStrategy; +import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; +import org.apache.cloudstack.engine.subsystem.api.storage.StrategyPriority; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService; +import org.apache.cloudstack.framework.async.AsyncCallFuture; +import org.apache.cloudstack.framework.async.AsyncCompletionCallback; +import org.apache.cloudstack.storage.command.CopyCmdAnswer; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.storage.datastore.util.LinstorUtil; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.stereotype.Component; + + +/** + * Current state: + * just changing the resource-group on same storage pool resource-group is not really good enough. + * Linstor lacks currently of a good way to move resources to another resource-group and respecting + * every auto-filter setting. + * Also linstor clone would simply set the new resource-group without any adjustments of storage pools or + * auto-select resource placement. + * So currently, we will create a new resource in the wanted primary storage and let qemu copy the data into the + * devices. + */ + +@Component +public class LinstorDataMotionStrategy implements DataMotionStrategy { + protected Logger logger = LogManager.getLogger(getClass()); + + @Inject + private SnapshotDataStoreDao _snapshotStoreDao; + @Inject + private PrimaryDataStoreDao _storagePool; + @Inject + private VolumeDao _volumeDao; + @Inject + private VolumeDataFactory _volumeDataFactory; + @Inject + private VMInstanceDao _vmDao; + @Inject + private GuestOSDao _guestOsDao; + @Inject + private VolumeService _volumeService; + @Inject + private GuestOSCategoryDao _guestOsCategoryDao; + @Inject + private SnapshotDao _snapshotDao; + @Inject + private AgentManager _agentManager; + @Inject + private PrimaryDataStoreDao _storagePoolDao; + + @Override + public StrategyPriority canHandle(DataObject srcData, DataObject dstData) { + DataObjectType srcType = srcData.getType(); + DataObjectType dstType = dstData.getType(); + logger.debug("canHandle: {} -> {}", srcType, dstType); + return StrategyPriority.CANT_HANDLE; + } + + @Override + public void copyAsync(DataObject srcData, DataObject destData, Host destHost, + AsyncCompletionCallback callback) { + throw new CloudRuntimeException("not implemented"); + } + + private boolean isDestinationLinstorPrimaryStorage(Map volumeMap) { + if (MapUtils.isNotEmpty(volumeMap)) { + for (DataStore dataStore : volumeMap.values()) { + StoragePoolVO storagePoolVO = _storagePool.findById(dataStore.getId()); + if (storagePoolVO == null + || !storagePoolVO.getStorageProviderName().equals(LinstorUtil.PROVIDER_NAME)) { + return false; + } + } + } else { + return false; + } + return true; + } + + @Override + public StrategyPriority canHandle(Map volumeMap, Host srcHost, Host destHost) { + logger.debug("canHandle -- {}: {} -> {}", volumeMap, srcHost, destHost); + if (srcHost.getId() != destHost.getId() && isDestinationLinstorPrimaryStorage(volumeMap)) { + return StrategyPriority.HIGHEST; + } + return StrategyPriority.CANT_HANDLE; + } + + private VolumeVO createNewVolumeVO(Volume volume, StoragePoolVO storagePoolVO) { + VolumeVO newVol = new VolumeVO(volume); + newVol.setInstanceId(null); + newVol.setChainInfo(null); + newVol.setPath(newVol.getUuid()); + newVol.setFolder(null); + newVol.setPodId(storagePoolVO.getPodId()); + newVol.setPoolId(storagePoolVO.getId()); + newVol.setLastPoolId(volume.getPoolId()); + + return _volumeDao.persist(newVol); + } + + private void removeExactSizeProperty(VolumeInfo volumeInfo) { + StoragePoolVO destStoragePool = _storagePool.findById(volumeInfo.getDataStore().getId()); + DevelopersApi api = LinstorUtil.getLinstorAPI(destStoragePool.getHostAddress()); + + ResourceDefinitionModify rdm = new ResourceDefinitionModify(); + rdm.setDeleteProps(Collections.singletonList(LinstorUtil.LIN_PROP_DRBDOPT_EXACT_SIZE)); + try { + String rscName = LinstorUtil.RSC_PREFIX + volumeInfo.getPath(); + ApiCallRcList answers = api.resourceDefinitionModify(rscName, rdm); + LinstorUtil.checkLinstorAnswersThrow(answers); + } catch (ApiException apiEx) { + logger.error("Linstor: ApiEx - {}", apiEx.getMessage()); + throw new CloudRuntimeException(apiEx.getBestMessage(), apiEx); + } + } + + private void handlePostMigration(boolean success, Map srcVolumeInfoToDestVolumeInfo, + VirtualMachineTO vmTO, Host destHost) { + if (!success) { + try { + PrepareForMigrationCommand pfmc = new PrepareForMigrationCommand(vmTO); + + pfmc.setRollback(true); + + Answer pfma = _agentManager.send(destHost.getId(), pfmc); + + if (pfma == null || !pfma.getResult()) { + String details = pfma != null ? pfma.getDetails() : "null answer returned"; + String msg = "Unable to rollback prepare for migration due to the following: " + details; + + throw new AgentUnavailableException(msg, destHost.getId()); + } + } catch (Exception e) { + logger.debug("Failed to disconnect one or more (original) dest volumes", e); + } + } + + for (Map.Entry entry : srcVolumeInfoToDestVolumeInfo.entrySet()) { + VolumeInfo srcVolumeInfo = entry.getKey(); + VolumeInfo destVolumeInfo = entry.getValue(); + + if (success) { + srcVolumeInfo.processEvent(ObjectInDataStoreStateMachine.Event.OperationSucceeded); + destVolumeInfo.processEvent(ObjectInDataStoreStateMachine.Event.OperationSucceeded); + + _volumeDao.updateUuid(srcVolumeInfo.getId(), destVolumeInfo.getId()); + + VolumeVO volumeVO = _volumeDao.findById(destVolumeInfo.getId()); + + volumeVO.setFormat(Storage.ImageFormat.QCOW2); + + _volumeDao.update(volumeVO.getId(), volumeVO); + + // remove exact size property + removeExactSizeProperty(destVolumeInfo); + + try { + _volumeService.destroyVolume(srcVolumeInfo.getId()); + + srcVolumeInfo = _volumeDataFactory.getVolume(srcVolumeInfo.getId()); + + AsyncCallFuture destroyFuture = + _volumeService.expungeVolumeAsync(srcVolumeInfo); + + if (destroyFuture.get().isFailed()) { + logger.debug("Failed to clean up source volume on storage"); + } + } catch (Exception e) { + logger.debug("Failed to clean up source volume on storage", e); + } + + // Update the volume ID for snapshots on secondary storage + if (!_snapshotDao.listByVolumeId(srcVolumeInfo.getId()).isEmpty()) { + _snapshotDao.updateVolumeIds(srcVolumeInfo.getId(), destVolumeInfo.getId()); + _snapshotStoreDao.updateVolumeIds(srcVolumeInfo.getId(), destVolumeInfo.getId()); + } + } else { + try { + _volumeService.revokeAccess(destVolumeInfo, destHost, destVolumeInfo.getDataStore()); + } catch (Exception e) { + logger.debug("Failed to revoke access from dest volume", e); + } + + destVolumeInfo.processEvent(ObjectInDataStoreStateMachine.Event.OperationFailed); + srcVolumeInfo.processEvent(ObjectInDataStoreStateMachine.Event.OperationFailed); + + try { + _volumeService.destroyVolume(destVolumeInfo.getId()); + + destVolumeInfo = _volumeDataFactory.getVolume(destVolumeInfo.getId()); + + AsyncCallFuture destroyFuture = + _volumeService.expungeVolumeAsync(destVolumeInfo); + + if (destroyFuture.get().isFailed()) { + logger.debug("Failed to clean up dest volume on storage"); + } + } catch (Exception e) { + logger.debug("Failed to clean up dest volume on storage", e); + } + } + } + } + + /** + * Determines whether the destination volume should have the DRBD exact-size property set + * during migration. + * + *

This method queries the Linstor API to check if the source volume's resource definition + * has the exact-size DRBD option enabled. The exact-size property ensures that DRBD uses + * the precise volume size rather than rounding, which is important for maintaining size + * consistency during migrations.

+ * + * @param srcVolumeInfo the source volume information to check + * @return {@code true} if the exact-size property should be set on the destination volume, + * which occurs when the source volume has this property enabled, or when the + * property cannot be determined (defaults to {@code true} for safety); + * {@code false} only when the source is confirmed to not have the exact-size property + */ + private boolean needsExactSizeProp(VolumeInfo srcVolumeInfo) { + StoragePoolVO srcStoragePool = _storagePool.findById(srcVolumeInfo.getDataStore().getId()); + if (srcStoragePool.getPoolType() == Storage.StoragePoolType.Linstor) { + DevelopersApi api = LinstorUtil.getLinstorAPI(srcStoragePool.getHostAddress()); + + String rscName = LinstorUtil.RSC_PREFIX + srcVolumeInfo.getPath(); + try { + List rscDfns = api.resourceDefinitionList( + Collections.singletonList(rscName), + false, + Collections.emptyList(), + null, + null); + if (!CollectionUtils.isEmpty(rscDfns)) { + ResourceDefinition srcRsc = rscDfns.get(0); + String exactSizeProp = srcRsc.getProps().get(LinstorUtil.LIN_PROP_DRBDOPT_EXACT_SIZE); + return "true".equalsIgnoreCase(exactSizeProp); + } else { + logger.warn("Unknown resource {} on {}", rscName, srcStoragePool.getHostAddress()); + } + } catch (ApiException apiEx) { + logger.error("Unable to fetch resource definition {}: {}", rscName, apiEx.getBestMessage()); + } + } + return true; + } + + @Override + public void copyAsync(Map volumeDataStoreMap, VirtualMachineTO vmTO, Host srcHost, + Host destHost, AsyncCompletionCallback callback) { + + if (srcHost.getHypervisorType() != Hypervisor.HypervisorType.KVM) { + throw new CloudRuntimeException( + String.format("Invalid hypervisor type [%s]. Only KVM supported", srcHost.getHypervisorType())); + } + + String errMsg = null; + VMInstanceVO vmInstance = _vmDao.findById(vmTO.getId()); + vmTO.setState(vmInstance.getState()); + List migrateDiskInfoList = new ArrayList<>(); + + Map migrateStorage = new HashMap<>(); + Map srcVolumeInfoToDestVolumeInfo = new HashMap<>(); + + try { + for (Map.Entry entry : volumeDataStoreMap.entrySet()) { + VolumeInfo srcVolumeInfo = entry.getKey(); + DataStore destDataStore = entry.getValue(); + VolumeVO srcVolume = _volumeDao.findById(srcVolumeInfo.getId()); + StoragePoolVO destStoragePool = _storagePool.findById(destDataStore.getId()); + + if (srcVolumeInfo.getPassphraseId() != null) { + throw new CloudRuntimeException( + String.format("Cannot live migrate encrypted volume: %s", srcVolumeInfo.getVolume())); + } + + VolumeVO destVolume = createNewVolumeVO(srcVolume, destStoragePool); + + VolumeInfo destVolumeInfo = _volumeDataFactory.getVolume(destVolume.getId(), destDataStore); + + destVolumeInfo.processEvent(ObjectInDataStoreStateMachine.Event.MigrationCopyRequested); + destVolumeInfo.processEvent(ObjectInDataStoreStateMachine.Event.MigrationCopySucceeded); + destVolumeInfo.processEvent(ObjectInDataStoreStateMachine.Event.MigrationRequested); + + boolean exactSize = needsExactSizeProp(srcVolumeInfo); + + String devPath = LinstorUtil.createResource( + destVolumeInfo, destStoragePool, _storagePoolDao, exactSize); + + _volumeDao.update(destVolume.getId(), destVolume); + destVolume = _volumeDao.findById(destVolume.getId()); + + destVolumeInfo = _volumeDataFactory.getVolume(destVolume.getId(), destDataStore); + + MigrateCommand.MigrateDiskInfo migrateDiskInfo = new MigrateCommand.MigrateDiskInfo( + srcVolumeInfo.getPath(), + MigrateCommand.MigrateDiskInfo.DiskType.BLOCK, + MigrateCommand.MigrateDiskInfo.DriverType.RAW, + MigrateCommand.MigrateDiskInfo.Source.DEV, + devPath); + migrateDiskInfoList.add(migrateDiskInfo); + + migrateStorage.put(srcVolumeInfo.getPath(), migrateDiskInfo); + + srcVolumeInfoToDestVolumeInfo.put(srcVolumeInfo, destVolumeInfo); + } + + PrepareForMigrationCommand pfmc = new PrepareForMigrationCommand(vmTO); + try { + Answer pfma = _agentManager.send(destHost.getId(), pfmc); + + if (pfma == null || !pfma.getResult()) { + String details = pfma != null ? pfma.getDetails() : "null answer returned"; + errMsg = String.format("Unable to prepare for migration due to the following: %s", details); + + throw new AgentUnavailableException(errMsg, destHost.getId()); + } + } catch (final OperationTimedoutException e) { + errMsg = String.format("Operation timed out due to %s", e.getMessage()); + throw new AgentUnavailableException(errMsg, destHost.getId()); + } + + VMInstanceVO vm = _vmDao.findById(vmTO.getId()); + boolean isWindows = _guestOsCategoryDao.findById(_guestOsDao.findById(vm.getGuestOSId()).getCategoryId()) + .getName().equalsIgnoreCase("Windows"); + + MigrateCommand migrateCommand = new MigrateCommand(vmTO.getName(), + destHost.getPrivateIpAddress(), isWindows, vmTO, true); + migrateCommand.setWait(StorageManager.KvmStorageOnlineMigrationWait.value()); + migrateCommand.setMigrateStorage(migrateStorage); + migrateCommand.setMigrateStorageManaged(true); + migrateCommand.setNewVmCpuShares( + vmTO.getCpus() * ObjectUtils.defaultIfNull(vmTO.getMinSpeed(), vmTO.getSpeed())); + migrateCommand.setMigrateDiskInfoList(migrateDiskInfoList); + + boolean kvmAutoConvergence = StorageManager.KvmAutoConvergence.value(); + migrateCommand.setAutoConvergence(kvmAutoConvergence); + + MigrateAnswer migrateAnswer = (MigrateAnswer) _agentManager.send(srcHost.getId(), migrateCommand); + boolean success = migrateAnswer != null && migrateAnswer.getResult(); + + handlePostMigration(success, srcVolumeInfoToDestVolumeInfo, vmTO, destHost); + + if (migrateAnswer == null) { + throw new CloudRuntimeException("Unable to get an answer to the migrate command"); + } + + if (!migrateAnswer.getResult()) { + errMsg = migrateAnswer.getDetails(); + + throw new CloudRuntimeException(errMsg); + } + } catch (AgentUnavailableException | OperationTimedoutException | CloudRuntimeException ex) { + errMsg = String.format( + "Copy volume(s) of VM [%s] to storage(s) [%s] and VM to host [%s] failed in LinstorDataMotionStrategy.copyAsync. Error message: [%s].", + vmTO, srcHost, destHost, ex.getMessage()); + logger.error(errMsg, ex); + + throw new CloudRuntimeException(errMsg); + } finally { + CopyCmdAnswer copyCmdAnswer = new CopyCmdAnswer(errMsg); + + CopyCommandResult result = new CopyCommandResult(null, copyCmdAnswer); + result.setResult(errMsg); + callback.complete(result); + } + } +} diff --git a/plugins/storage/volume/linstor/src/main/resources/META-INF/cloudstack/storage-volume-linstor/spring-storage-volume-linstor-context.xml b/plugins/storage/volume/linstor/src/main/resources/META-INF/cloudstack/storage-volume-linstor/spring-storage-volume-linstor-context.xml index a900323ede5..88d1051c71e 100644 --- a/plugins/storage/volume/linstor/src/main/resources/META-INF/cloudstack/storage-volume-linstor/spring-storage-volume-linstor-context.xml +++ b/plugins/storage/volume/linstor/src/main/resources/META-INF/cloudstack/storage-volume-linstor/spring-storage-volume-linstor-context.xml @@ -33,4 +33,6 @@ class="org.apache.cloudstack.storage.snapshot.LinstorVMSnapshotStrategy" /> + diff --git a/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImplTest.java b/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImplTest.java index 75276739468..4653cfa358b 100644 --- a/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImplTest.java +++ b/plugins/storage/volume/linstor/src/test/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImplTest.java @@ -26,6 +26,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import org.apache.cloudstack.storage.datastore.util.LinstorUtil; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -75,13 +76,13 @@ public class LinstorPrimaryDataStoreDriverImplTest { when(api.resourceGroupList(Collections.singletonList("EncryptedGrp"), Collections.emptyList(), null, null)) .thenReturn(Collections.singletonList(encryptedGrp)); - List layers = linstorPrimaryDataStoreDriver.getEncryptedLayerList(api, "DfltRscGrp"); + List layers = LinstorUtil.getEncryptedLayerList(api, "DfltRscGrp"); Assert.assertEquals(Arrays.asList(LayerType.DRBD, LayerType.LUKS, LayerType.STORAGE), layers); - layers = linstorPrimaryDataStoreDriver.getEncryptedLayerList(api, "BcacheGrp"); + layers = LinstorUtil.getEncryptedLayerList(api, "BcacheGrp"); Assert.assertEquals(Arrays.asList(LayerType.DRBD, LayerType.BCACHE, LayerType.LUKS, LayerType.STORAGE), layers); - layers = linstorPrimaryDataStoreDriver.getEncryptedLayerList(api, "EncryptedGrp"); + layers = LinstorUtil.getEncryptedLayerList(api, "EncryptedGrp"); Assert.assertEquals(Arrays.asList(LayerType.DRBD, LayerType.LUKS, LayerType.STORAGE), layers); } } diff --git a/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/driver/SolidFirePrimaryDataStoreDriver.java b/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/driver/SolidFirePrimaryDataStoreDriver.java index 6cc76d99d9e..1e927e20168 100644 --- a/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/driver/SolidFirePrimaryDataStoreDriver.java +++ b/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/driver/SolidFirePrimaryDataStoreDriver.java @@ -433,7 +433,7 @@ public class SolidFirePrimaryDataStoreDriver implements PrimaryDataStoreDriver { public long getUsedIops(StoragePool storagePool) { long usedIops = 0; - List volumes = volumeDao.findByPoolId(storagePool.getId(), null); + List volumes = volumeDao.findNonDestroyedVolumesByPoolId(storagePool.getId(), null); if (volumes != null) { for (VolumeVO volume : volumes) { diff --git a/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/provider/SolidFireHostListener.java b/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/provider/SolidFireHostListener.java index 052191128f1..c961c926739 100644 --- a/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/provider/SolidFireHostListener.java +++ b/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/provider/SolidFireHostListener.java @@ -199,7 +199,7 @@ public class SolidFireHostListener implements HypervisorHostListener { List storagePaths = new ArrayList<>(); // If you do not pass in null for the second parameter, you only get back applicable ROOT disks. - List volumes = volumeDao.findByPoolId(storagePoolId, null); + List volumes = volumeDao.findNonDestroyedVolumesByPoolId(storagePoolId, null); if (volumes != null) { for (VolumeVO volume : volumes) { @@ -230,7 +230,7 @@ public class SolidFireHostListener implements HypervisorHostListener { StoragePoolVO storagePool = storagePoolDao.findById(storagePoolId); // If you do not pass in null for the second parameter, you only get back applicable ROOT disks. - List volumes = volumeDao.findByPoolId(storagePoolId, null); + List volumes = volumeDao.findNonDestroyedVolumesByPoolId(storagePoolId, null); if (volumes != null) { for (VolumeVO volume : volumes) { diff --git a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/driver/StorPoolPrimaryDataStoreDriver.java b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/driver/StorPoolPrimaryDataStoreDriver.java index ba9a4d97600..9901b4cb69d 100644 --- a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/driver/StorPoolPrimaryDataStoreDriver.java +++ b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/driver/StorPoolPrimaryDataStoreDriver.java @@ -1276,7 +1276,7 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver { return volumeStats; } } else { - List volumes = volumeDao.findByPoolId(storagePool.getId()); + List volumes = volumeDao.findNonDestroyedVolumesByPoolId(storagePool.getId()); for (VolumeVO volume : volumes) { if (volume.getPath() != null && volume.getPath().equals(volumeId)) { long size = volume.getSize(); diff --git a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/util/StorPoolHelper.java b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/util/StorPoolHelper.java index f13d296af3b..e6b70c7123e 100644 --- a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/util/StorPoolHelper.java +++ b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/util/StorPoolHelper.java @@ -102,7 +102,7 @@ public class StorPoolHelper { if (snapshotDetails != null) { return StorPoolStorageAdaptor.getVolumeNameFromPath(snapshotDetails.getValue(), true); } else { - List snapshots = snapshotStoreDao.findBySnapshotId(snapshotId); + List snapshots = snapshotStoreDao.findBySnapshotIdWithNonDestroyedState(snapshotId); if (!CollectionUtils.isEmpty(snapshots)) { for (SnapshotDataStoreVO snapshotDataStoreVO : snapshots) { String name = StorPoolStorageAdaptor.getVolumeNameFromPath(snapshotDataStoreVO.getInstallPath(), true); diff --git a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/snapshot/StorPoolSnapshotStrategy.java b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/snapshot/StorPoolSnapshotStrategy.java index 8a51e2672ea..b343848980b 100644 --- a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/snapshot/StorPoolSnapshotStrategy.java +++ b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/snapshot/StorPoolSnapshotStrategy.java @@ -240,7 +240,7 @@ public class StorPoolSnapshotStrategy implements SnapshotStrategy { } protected boolean areLastSnapshotRef(long snapshotId) { - List snapshotStoreRefs = _snapshotStoreDao.findBySnapshotId(snapshotId); + List snapshotStoreRefs = _snapshotStoreDao.findBySnapshotIdWithNonDestroyedState(snapshotId); if (CollectionUtils.isEmpty(snapshotStoreRefs) || snapshotStoreRefs.size() == 1) { return true; } @@ -308,7 +308,7 @@ public class StorPoolSnapshotStrategy implements SnapshotStrategy { } if (Snapshot.State.Error.equals(snapshotVO.getState())) { - List storeRefs = _snapshotStoreDao.findBySnapshotId(snapshotId); + List storeRefs = _snapshotStoreDao.findBySnapshotIdWithNonDestroyedState(snapshotId); List deletedRefs = new ArrayList<>(); for (SnapshotDataStoreVO ref : storeRefs) { boolean refZoneIdMatch = false; diff --git a/plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/saml/SAML2AuthManager.java b/plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/saml/SAML2AuthManager.java index 523f694d80b..618536a71f6 100644 --- a/plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/saml/SAML2AuthManager.java +++ b/plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/saml/SAML2AuthManager.java @@ -82,6 +82,9 @@ public interface SAML2AuthManager extends PluggableAPIAuthenticator, PluggableSe ConfigKey SAMLRequirePasswordLogin = new ConfigKey("Advanced", Boolean.class, "saml2.require.password", "true", "When enabled SAML2 will validate that the SAML login was performed with a password. If disabled, other forms of authentication are allowed (two-factor, certificate, etc) on the SAML Authentication Provider", true); + ConfigKey EnableLoginAfterSAMLDisable = new ConfigKey<>("Advanced", Boolean.class, "enable.login.with.disabled.saml", "false", "When enabled, if SAML SSO is disabled, enables user to login with user and password, otherwise a user with SAML SSO disabled cannot login", true); + + SAMLProviderMetadata getSPMetadata(); SAMLProviderMetadata getIdPMetadata(String entityId); diff --git a/plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/saml/SAML2AuthManagerImpl.java b/plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/saml/SAML2AuthManagerImpl.java index 93b7bc5be93..9f8101b867d 100644 --- a/plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/saml/SAML2AuthManagerImpl.java +++ b/plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/saml/SAML2AuthManagerImpl.java @@ -451,8 +451,13 @@ public class SAML2AuthManagerImpl extends AdapterBase implements SAML2AuthManage user.setExternalEntity(entityId); user.setSource(User.Source.SAML2); } else { + boolean enableLoginAfterSAMLDisable = SAML2AuthManager.EnableLoginAfterSAMLDisable.value(); if (user.getSource().equals(User.Source.SAML2)) { - user.setSource(User.Source.SAML2DISABLED); + if(enableLoginAfterSAMLDisable) { + user.setSource(User.Source.UNKNOWN); + } else { + user.setSource(User.Source.SAML2DISABLED); + } } else { return false; } @@ -541,6 +546,6 @@ public class SAML2AuthManagerImpl extends AdapterBase implements SAML2AuthManage SAMLCloudStackRedirectionUrl, SAMLUserAttributeName, SAMLIdentityProviderMetadataURL, SAMLDefaultIdentityProviderId, SAMLSignatureAlgorithm, SAMLAppendDomainSuffix, SAMLTimeout, SAMLCheckSignature, - SAMLForceAuthn, SAMLUserSessionKeyPathAttribute, SAMLRequirePasswordLogin}; + SAMLForceAuthn, SAMLUserSessionKeyPathAttribute, SAMLRequirePasswordLogin, EnableLoginAfterSAMLDisable}; } } diff --git a/scripts/storage/secondary/cloud-install-sys-tmplt b/scripts/storage/secondary/cloud-install-sys-tmplt index ad976c502c6..fc09dc968ff 100755 --- a/scripts/storage/secondary/cloud-install-sys-tmplt +++ b/scripts/storage/secondary/cloud-install-sys-tmplt @@ -44,6 +44,7 @@ failed() { } #set -x +umask 0022 # ensure we have the proper permissions even on hardened deployments mflag= fflag= ext="vhd" diff --git a/scripts/storage/secondary/setup-sysvm-tmplt b/scripts/storage/secondary/setup-sysvm-tmplt index 06f0586fe34..63006cc4e4c 100755 --- a/scripts/storage/secondary/setup-sysvm-tmplt +++ b/scripts/storage/secondary/setup-sysvm-tmplt @@ -19,6 +19,7 @@ # Usage: e.g. failed $? "this is an error" set -x +umask 0022 # ensure we have the proper permissions even on hardened deployments failed() { local returnval=$1 diff --git a/server/src/main/java/com/cloud/alert/AlertManagerImpl.java b/server/src/main/java/com/cloud/alert/AlertManagerImpl.java index 377b2134d78..7bf00037ee4 100644 --- a/server/src/main/java/com/cloud/alert/AlertManagerImpl.java +++ b/server/src/main/java/com/cloud/alert/AlertManagerImpl.java @@ -294,13 +294,8 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager, Confi Math.min(CapacityManager.CapacityCalculateWorkers.value(), hostIds.size()))); for (Long hostId : hostIds) { futures.put(hostId, executorService.submit(() -> { - Transaction.execute(new TransactionCallbackNoReturn() { - @Override - public void doInTransactionWithoutResult(TransactionStatus status) { - final HostVO host = hostDao.findById(hostId); - _capacityMgr.updateCapacityForHost(host); - } - }); + final HostVO host = hostDao.findById(hostId); + _capacityMgr.updateCapacityForHost(host); return null; })); } diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java index 5833ede550e..a89a1eed87b 100644 --- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java @@ -2309,6 +2309,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q Long pageSize = cmd.getPageSizeVal(); Hypervisor.HypervisorType hypervisorType = cmd.getHypervisor(); final CPU.CPUArch arch = cmd.getArch(); + String version = cmd.getVersion(); Filter searchFilter = new Filter(HostVO.class, "id", Boolean.TRUE, startIndex, pageSize); @@ -2325,11 +2326,13 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q hostSearchBuilder.and("resourceState", hostSearchBuilder.entity().getResourceState(), SearchCriteria.Op.EQ); hostSearchBuilder.and("hypervisor_type", hostSearchBuilder.entity().getHypervisorType(), SearchCriteria.Op.EQ); hostSearchBuilder.and("arch", hostSearchBuilder.entity().getArch(), SearchCriteria.Op.EQ); + hostSearchBuilder.and("version", hostSearchBuilder.entity().getVersion(), SearchCriteria.Op.EQ); if (keyword != null) { hostSearchBuilder.and().op("keywordName", hostSearchBuilder.entity().getName(), SearchCriteria.Op.LIKE); hostSearchBuilder.or("keywordStatus", hostSearchBuilder.entity().getStatus(), SearchCriteria.Op.LIKE); hostSearchBuilder.or("keywordType", hostSearchBuilder.entity().getType(), SearchCriteria.Op.LIKE); + hostSearchBuilder.or("keywordVersion", hostSearchBuilder.entity().getVersion(), SearchCriteria.Op.LIKE); hostSearchBuilder.cp(); } @@ -2360,6 +2363,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q sc.setParameters("keywordName", "%" + keyword + "%"); sc.setParameters("keywordStatus", "%" + keyword + "%"); sc.setParameters("keywordType", "%" + keyword + "%"); + sc.setParameters("keywordVersion", "%" + keyword + "%"); } if (id != null) { @@ -2409,6 +2413,10 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q sc.setParameters("arch", arch); } + if (version != null) { + sc.setParameters("version", version); + } + Pair, Integer> uniqueHostPair = hostDao.searchAndCount(sc, searchFilter); Integer count = uniqueHostPair.second(); List hostIds = uniqueHostPair.first().stream().map(HostVO::getId).collect(Collectors.toList()); @@ -5397,6 +5405,8 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q protected Pair, Integer> listManagementServersInternal(ListMgmtsCmd cmd) { Long id = cmd.getId(); String name = cmd.getHostName(); + String version = cmd.getVersion(); + String keyword = cmd.getKeyword(); SearchBuilder sb = managementServerJoinDao.createSearchBuilder(); SearchCriteria sc = sb.create(); @@ -5406,6 +5416,12 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q if (name != null) { sc.addAnd("name", SearchCriteria.Op.EQ, name); } + if (version != null) { + sc.addAnd("version", SearchCriteria.Op.EQ, version); + } + if (keyword != null) { + sc.addAnd("version", SearchCriteria.Op.LIKE, "%" + keyword + "%"); + } return managementServerJoinDao.searchAndCount(sc, null); } diff --git a/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDaoImpl.java index e7da4a2163b..41d516f6dd3 100644 --- a/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDaoImpl.java @@ -78,6 +78,7 @@ import com.cloud.template.VirtualMachineTemplate; import com.cloud.user.Account; import com.cloud.user.AccountService; import com.cloud.utils.Pair; +import com.cloud.utils.StringUtils; import com.cloud.utils.db.Filter; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; @@ -162,29 +163,50 @@ public class TemplateJoinDaoImpl extends GenericDaoBaseWithTagInformation isosInStore = _templateStoreDao.listByTemplateNotBypassed(iso.getId()); diff --git a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java index e662d154f65..88d64198559 100644 --- a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java +++ b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java @@ -527,7 +527,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati public static final ConfigKey DELETE_QUERY_BATCH_SIZE = new ConfigKey<>("Advanced", Long.class, "delete.query.batch.size", "0", "Indicates the limit applied while deleting entries in bulk. With this, the delete query will apply the limit as many times as necessary," + " to delete all the entries. This is advised when retaining several days of records, which can lead to slowness. <= 0 means that no limit will " + - "be applied. Default value is 0. For now, this is used for deletion of vm & volume stats only.", true); + "be applied. Default value is 0. For now, this is used for deletion of VM stats, volume stats, and usage records.", true); private static final String IOPS_READ_RATE = "IOPS Read"; private static final String IOPS_WRITE_RATE = "IOPS Write"; diff --git a/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java b/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java index e7b926eb4e4..6881fbab98c 100644 --- a/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java +++ b/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java @@ -36,6 +36,7 @@ import java.util.stream.Collectors; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.resource.ResourceState; import org.apache.cloudstack.affinity.AffinityGroupDomainMapVO; import org.apache.cloudstack.affinity.AffinityGroupProcessor; import org.apache.cloudstack.affinity.AffinityGroupService; @@ -378,22 +379,12 @@ StateListener, Configurable { planner = getDeploymentPlannerByName(plannerName); } - Host lastHost = null; - - String considerLastHostStr = (String)vmProfile.getParameter(VirtualMachineProfile.Param.ConsiderLastHost); - boolean considerLastHost = vm.getLastHostId() != null && haVmTag == null && - (considerLastHostStr == null || Boolean.TRUE.toString().equalsIgnoreCase(considerLastHostStr)); - if (considerLastHost) { - HostVO host = _hostDao.findById(vm.getLastHostId()); - logger.debug("This VM has last host_id specified, trying to choose the same host: " + host); - lastHost = host; - - DeployDestination deployDestination = deployInVmLastHost(vmProfile, plan, avoids, planner, vm, dc, offering, cpuRequested, ramRequested, volumesRequireEncryption); - if (deployDestination != null) { - return deployDestination; - } + DeployDestination deployDestinationForVmLasthost = deployInVmLastHost(vmProfile, plan, avoids, planner, vm, dc, offering, cpuRequested, ramRequested, volumesRequireEncryption); + if (deployDestinationForVmLasthost != null) { + return deployDestinationForVmLasthost; } + HostVO lastHost = _hostDao.findById(vm.getLastHostId()); avoidOtherClustersForDeploymentIfMigrationDisabled(vm, lastHost, avoids); DeployDestination dest = null; @@ -475,47 +466,56 @@ StateListener, Configurable { private DeployDestination deployInVmLastHost(VirtualMachineProfile vmProfile, DeploymentPlan plan, ExcludeList avoids, DeploymentPlanner planner, VirtualMachine vm, DataCenter dc, ServiceOffering offering, int cpuRequested, long ramRequested, boolean volumesRequireEncryption) throws InsufficientServerCapacityException { - HostVO host = _hostDao.findById(vm.getLastHostId()); - if (canUseLastHost(host, avoids, plan, vm, offering, volumesRequireEncryption)) { - _hostDao.loadHostTags(host); - _hostDao.loadDetails(host); - if (host.getStatus() != Status.Up) { + String considerLastHostStr = (String)vmProfile.getParameter(VirtualMachineProfile.Param.ConsiderLastHost); + String haVmTag = (String)vmProfile.getParameter(VirtualMachineProfile.Param.HaTag); + boolean considerLastHost = vm.getLastHostId() != null && haVmTag == null && + !(Boolean.FALSE.toString().equalsIgnoreCase(considerLastHostStr)); + if (!considerLastHost) { + return null; + } + + logger.debug("This VM has last host_id: {}", vm.getLastHostId()); + HostVO lastHost = _hostDao.findById(vm.getLastHostId()); + if (canUseLastHost(lastHost, avoids, plan, vm, offering, volumesRequireEncryption)) { + _hostDao.loadHostTags(lastHost); + _hostDao.loadDetails(lastHost); + if (lastHost.getStatus() != Status.Up) { logger.debug("Cannot deploy VM [{}] to the last host [{}] because this host is not in UP state or is not enabled. Host current status [{}] and resource status [{}].", - vm, host, host.getState().name(), host.getResourceState()); + vm, lastHost, lastHost.getState().name(), lastHost.getResourceState()); return null; } - if (checkVmProfileAndHost(vmProfile, host)) { - long cluster_id = host.getClusterId(); + if (checkVmProfileAndHost(vmProfile, lastHost)) { + long cluster_id = lastHost.getClusterId(); ClusterDetailsVO cluster_detail_cpu = _clusterDetailsDao.findDetail(cluster_id, "cpuOvercommitRatio"); ClusterDetailsVO cluster_detail_ram = _clusterDetailsDao.findDetail(cluster_id, "memoryOvercommitRatio"); float cpuOvercommitRatio = Float.parseFloat(cluster_detail_cpu.getValue()); float memoryOvercommitRatio = Float.parseFloat(cluster_detail_ram.getValue()); boolean hostHasCpuCapability, hostHasCapacity = false; - hostHasCpuCapability = _capacityMgr.checkIfHostHasCpuCapability(host, offering.getCpu(), offering.getSpeed()); + hostHasCpuCapability = _capacityMgr.checkIfHostHasCpuCapability(lastHost, offering.getCpu(), offering.getSpeed()); if (hostHasCpuCapability) { // first check from reserved capacity - hostHasCapacity = _capacityMgr.checkIfHostHasCapacity(host, cpuRequested, ramRequested, true, cpuOvercommitRatio, memoryOvercommitRatio, true); + hostHasCapacity = _capacityMgr.checkIfHostHasCapacity(lastHost, cpuRequested, ramRequested, true, cpuOvercommitRatio, memoryOvercommitRatio, true); // if not reserved, check the free capacity if (!hostHasCapacity) - hostHasCapacity = _capacityMgr.checkIfHostHasCapacity(host, cpuRequested, ramRequested, false, cpuOvercommitRatio, memoryOvercommitRatio, true); + hostHasCapacity = _capacityMgr.checkIfHostHasCapacity(lastHost, cpuRequested, ramRequested, false, cpuOvercommitRatio, memoryOvercommitRatio, true); } boolean displayStorage = getDisplayStorageFromVmProfile(vmProfile); if (!hostHasCapacity || !hostHasCpuCapability) { - logger.debug("Cannot deploy VM [{}] to the last host [{}] because this host does not have enough capacity to deploy this VM.", vm, host); + logger.debug("Cannot deploy VM [{}] to the last host [{}] because this host does not have enough capacity to deploy this VM.", vm, lastHost); return null; } - Pod pod = _podDao.findById(host.getPodId()); - Cluster cluster = _clusterDao.findById(host.getClusterId()); + Pod pod = _podDao.findById(lastHost.getPodId()); + Cluster cluster = _clusterDao.findById(lastHost.getClusterId()); logger.debug("Last host [{}] of VM [{}] is UP and has enough capacity. Checking for suitable pools for this host under zone [{}], pod [{}] and cluster [{}].", - host, vm, dc, pod, cluster); + lastHost, vm, dc, pod, cluster); if (vm.getHypervisorType() == HypervisorType.BareMetal) { - DeployDestination dest = new DeployDestination(dc, pod, cluster, host, new HashMap<>(), displayStorage); + DeployDestination dest = new DeployDestination(dc, pod, cluster, lastHost, new HashMap<>(), displayStorage); logger.debug("Returning Deployment Destination: {}.", dest); return dest; } @@ -523,8 +523,8 @@ StateListener, Configurable { // search for storage under the zone, pod, cluster // of // the last host. - DataCenterDeployment lastPlan = new DataCenterDeployment(host.getDataCenterId(), - host.getPodId(), host.getClusterId(), host.getId(), plan.getPoolId(), null); + DataCenterDeployment lastPlan = new DataCenterDeployment(lastHost.getDataCenterId(), + lastHost.getPodId(), lastHost.getClusterId(), lastHost.getId(), plan.getPoolId(), null); Pair>, List> result = findSuitablePoolsForVolumes( vmProfile, lastPlan, avoids, HostAllocator.RETURN_UPTO_ALL); Map> suitableVolumeStoragePools = result.first(); @@ -533,11 +533,11 @@ StateListener, Configurable { // choose the potential pool for this VM for this // host if (suitableVolumeStoragePools.isEmpty()) { - logger.debug("Cannot find suitable storage pools in host [{}] to deploy VM [{}]", host, vm); + logger.debug("Cannot find suitable storage pools in host [{}] to deploy VM [{}]", lastHost, vm); return null; } List suitableHosts = new ArrayList<>(); - suitableHosts.add(host); + suitableHosts.add(lastHost); Pair> potentialResources = findPotentialDeploymentResources( suitableHosts, suitableVolumeStoragePools, avoids, getPlannerUsage(planner, vmProfile, plan, avoids), readyAndReusedVolumes, plan.getPreferredHosts(), vm); @@ -550,7 +550,7 @@ StateListener, Configurable { for (Volume vol : readyAndReusedVolumes) { storageVolMap.remove(vol); } - DeployDestination dest = new DeployDestination(dc, pod, cluster, host, storageVolMap, displayStorage); + DeployDestination dest = new DeployDestination(dc, pod, cluster, lastHost, storageVolMap, displayStorage); logger.debug("Returning Deployment Destination: {}", dest); return dest; } @@ -562,7 +562,7 @@ StateListener, Configurable { private boolean canUseLastHost(HostVO host, ExcludeList avoids, DeploymentPlan plan, VirtualMachine vm, ServiceOffering offering, boolean volumesRequireEncryption) { if (host == null) { - logger.warn("Could not find last host of VM [{}] with id [{}]. Skipping this and trying other available hosts.", vm, vm.getLastHostId()); + logger.warn("Could not find last host of VM [{}] with id [{}]. Skipping it", vm, vm.getLastHostId()); return false; } @@ -576,6 +576,12 @@ StateListener, Configurable { return false; } + logger.debug("VM's last host is {}, trying to choose the same host if it is not in maintenance, error or degraded state", host); + if (host.isInMaintenanceStates() || Arrays.asList(ResourceState.Error, ResourceState.Degraded).contains(host.getResourceState())) { + logger.debug("Unable to deploy VM {} in the last host, last host {} is in {} state", vm.getName(), host.getName(), host.getResourceState()); + return false; + } + if (_capacityMgr.checkIfHostReachMaxGuestLimit(host)) { logger.debug("Cannot deploy VM [{}] in the last host [{}] because this host already has the max number of running VMs (users and system VMs). Skipping this and trying other available hosts.", vm, host); @@ -1474,7 +1480,7 @@ StateListener, Configurable { protected Pair> findPotentialDeploymentResources(List suitableHosts, Map> suitableVolumeStoragePools, ExcludeList avoid, PlannerResourceUsage resourceUsageRequired, List readyAndReusedVolumes, List preferredHosts, VirtualMachine vm) { - logger.debug("Trying to find a potenial host and associated storage pools from the suitable host/pool lists for this VM"); + logger.debug("Trying to find a potential host and associated storage pools from the suitable host/pool lists for this VM"); boolean hostCanAccessPool = false; boolean haveEnoughSpace = false; diff --git a/server/src/main/java/com/cloud/event/AlertGenerator.java b/server/src/main/java/com/cloud/event/AlertGenerator.java index f1b23e87308..601bf5e831a 100644 --- a/server/src/main/java/com/cloud/event/AlertGenerator.java +++ b/server/src/main/java/com/cloud/event/AlertGenerator.java @@ -67,12 +67,13 @@ public class AlertGenerator { } public static void publishAlertOnEventBus(String alertType, long dataCenterId, Long podId, String subject, String body) { - String configKey = Config.PublishAlertEvent.key(); String value = s_configDao.getValue(configKey); boolean configValue = Boolean.parseBoolean(value); - if(!configValue) + if (!configValue) { return; + } + try { eventDistributor = ComponentContext.getComponent(EventDistributor.class); } catch (NoSuchBeanDefinitionException nbe) { diff --git a/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java b/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java index daf1e0faa03..82a723a4898 100644 --- a/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java +++ b/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java @@ -833,7 +833,7 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements Configur if (checkAndCancelWorkIfNeeded(work)) { return null; } - logger.info("Migration attempt: for VM {}from host {}. Starting attempt: {}/{} times.", vm, srcHost, 1 + work.getTimesTried(), _maxRetries); + logger.info("Migration attempt: for {} from {}. Starting attempt: {}/{} times.", vm, srcHost, 1 + work.getTimesTried(), _maxRetries); if (VirtualMachine.State.Stopped.equals(vm.getState())) { logger.info(String.format("vm %s is Stopped, skipping migrate.", vm)); @@ -843,8 +843,6 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements Configur logger.info(String.format("VM %s is running on a different host %s, skipping migration", vm, vm.getHostId())); return null; } - logger.info("Migration attempt: for VM " + vm.getUuid() + "from host id " + srcHostId + - ". Starting attempt: " + (1 + work.getTimesTried()) + "/" + _maxRetries + " times."); try { work.setStep(Step.Migrating); diff --git a/server/src/main/java/com/cloud/network/NetworkModelImpl.java b/server/src/main/java/com/cloud/network/NetworkModelImpl.java index 4a5b7199430..86791b87851 100644 --- a/server/src/main/java/com/cloud/network/NetworkModelImpl.java +++ b/server/src/main/java/com/cloud/network/NetworkModelImpl.java @@ -1442,11 +1442,11 @@ public class NetworkModelImpl extends ManagerBase implements NetworkModel, Confi return null; } + NetworkOffering offering = _entityMgr.findById(NetworkOffering.class, network.getNetworkOfferingId()); Long physicalNetworkId = null; if (effectiveTrafficType != TrafficType.Guest) { physicalNetworkId = getNonGuestNetworkPhysicalNetworkId(network, effectiveTrafficType); } else { - NetworkOffering offering = _entityMgr.findById(NetworkOffering.class, network.getNetworkOfferingId()); physicalNetworkId = network.getPhysicalNetworkId(); if (physicalNetworkId == null) { physicalNetworkId = findPhysicalNetworkId(network.getDataCenterId(), offering.getTags(), offering.getTrafficType()); @@ -1459,6 +1459,10 @@ public class NetworkModelImpl extends ManagerBase implements NetworkModel, Confi return null; } + if (offering != null && TrafficType.Guest.equals(offering.getTrafficType()) && offering.isSystemOnly()) { + // For private gateway, do not check the Guest traffic type + return _pNTrafficTypeDao.getNetworkTag(physicalNetworkId, null, hType); + } return _pNTrafficTypeDao.getNetworkTag(physicalNetworkId, effectiveTrafficType, hType); } diff --git a/server/src/main/java/com/cloud/network/NetworkServiceImpl.java b/server/src/main/java/com/cloud/network/NetworkServiceImpl.java index 3c5b7ec7b1f..cfa4574d996 100644 --- a/server/src/main/java/com/cloud/network/NetworkServiceImpl.java +++ b/server/src/main/java/com/cloud/network/NetworkServiceImpl.java @@ -1869,6 +1869,18 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C null, null, null, null, null, null, null, null, null, null); } + @Override + @DB + @ActionEvent(eventType = EventTypes.EVENT_NETWORK_CREATE, eventDescription = "creating network") + public Network createGuestNetwork(long networkOfferingId, String name, String displayText, Account owner, + PhysicalNetwork physicalNetwork, long zoneId, ACLType aclType, Pair vrIfaceMTUs) throws + InsufficientCapacityException, ConcurrentOperationException, ResourceAllocationException { + return _networkMgr.createGuestNetwork(networkOfferingId, name, displayText, + null, null, null, false, null, owner, null, physicalNetwork, zoneId, + aclType, null, null, null, null, true, null, + null, null, null, null, null, null, null, null, vrIfaceMTUs, null); + } + void checkAndSetRouterSourceNatIp(Account owner, CreateNetworkCmd cmd, Network network) throws InsufficientAddressCapacityException, ResourceAllocationException { String sourceNatIp = cmd.getSourceNatIP(); if (sourceNatIp == null) { diff --git a/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java b/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java index e171b68399b..bd73e67f10b 100644 --- a/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java +++ b/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java @@ -1777,8 +1777,9 @@ Configurable, StateListener vms = userVmJoinDao.search(scvm, null); - boolean isDhcpSupported = _ntwkSrvcDao.areServicesSupportedInNetwork(routerJoinVO.getNetworkId(), Service.Dhcp); - boolean isDnsSupported = _ntwkSrvcDao.areServicesSupportedInNetwork(routerJoinVO.getNetworkId(), Service.Dns); + Provider provider = routerJoinVO.getVpcId() != 0 ? Provider.VPCVirtualRouter : Provider.VirtualRouter; + boolean isDhcpSupported = _ntwkSrvcDao.canProviderSupportServiceInNetwork(routerJoinVO.getNetworkId(), Service.Dhcp, provider); + boolean isDnsSupported = _ntwkSrvcDao.canProviderSupportServiceInNetwork(routerJoinVO.getNetworkId(), Service.Dns, provider); for (UserVmJoinVO vm : vms) { vmsData.append("vmName=").append(vm.getName()) .append(",macAddress=").append(vm.getMacAddress()) @@ -3358,6 +3359,7 @@ Configurable, StateListener rootDisks = volumeDao.findByPoolId(poolId); - List dataVolumes = volumeDao.findByPoolId(poolId, Volume.Type.DATADISK); + List rootDisks = volumeDao.findNonDestroyedVolumesByPoolId(poolId); + List dataVolumes = volumeDao.findNonDestroyedVolumesByPoolId(poolId, Volume.Type.DATADISK); List volumes = new ArrayList<>(); addVolumesToList(volumes, rootDisks); @@ -1417,7 +1417,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, throw new CloudRuntimeException("There are active VMs using the host's local storage pool. Please stop all VMs on this host that use local storage."); } } else { - logger.info("Maintenance: scheduling migration of VM {} from host {}", vm, host); + logger.info("Maintenance: scheduling migration of {} from {}", vm, host); _haMgr.scheduleMigration(vm, HighAvailabilityManager.ReasonType.HostMaintenance); } } @@ -2920,8 +2920,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, if (!isAgentOnHost || vmsMigrating || host.getStatus() == Status.Up) { return; } - final boolean sshToAgent = Boolean.parseBoolean(_configDao.getValue(KvmSshToAgentEnabled.key())); - if (sshToAgent) { + if (KvmSshToAgentEnabled.value()) { Ternary credentials = getHostCredentials(host); connectAndRestartAgentOnHost(host, credentials.first(), credentials.second(), credentials.third()); } else { diff --git a/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java b/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java index b890b72f758..d4d91b6de7b 100644 --- a/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java +++ b/server/src/main/java/com/cloud/resourcelimit/ResourceLimitManagerImpl.java @@ -1171,7 +1171,6 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim } return Transaction.execute((TransactionCallback) status -> { - long newResourceCount = 0L; List domainIdList = childDomains.stream().map(DomainVO::getId).collect(Collectors.toList()); domainIdList.add(domainId); List accountIdList = accounts.stream().map(AccountVO::getId).collect(Collectors.toList()); @@ -1189,6 +1188,7 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim List resourceCounts = _resourceCountDao.lockRows(rowIdsToLock); long oldResourceCount = 0L; + long newResourceCount = 0L; ResourceCountVO domainRC = null; // calculate project count here @@ -1210,7 +1210,7 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim if (oldResourceCount != newResourceCount) { domainRC.setCount(newResourceCount); _resourceCountDao.update(domainRC.getId(), domainRC); - logger.warn("Discrepency in the resource count has been detected (original count = {} correct count = {}) for Type = {} for Domain ID = {} is fixed during resource count recalculation.", + logger.warn("Discrepancy in the resource count has been detected (original count = {} correct count = {}) for Type = {} for Domain ID = {} is fixed during resource count recalculation.", oldResourceCount, newResourceCount, type, domainId); } return newResourceCount; @@ -1436,16 +1436,17 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim } protected long calculatePrimaryStorageForAccount(long accountId, String tag) { + long snapshotsPhysicalSizeOnPrimaryStorage = _snapshotDataStoreDao.getSnapshotsPhysicalSizeOnPrimaryStorageByAccountId(accountId); if (StringUtils.isEmpty(tag)) { List virtualRouters = _vmDao.findIdsOfAllocatedVirtualRoutersForAccount(accountId); - return _volumeDao.primaryStorageUsedForAccount(accountId, virtualRouters); + return snapshotsPhysicalSizeOnPrimaryStorage + _volumeDao.primaryStorageUsedForAccount(accountId, virtualRouters); } long storage = 0; List volumes = getVolumesWithAccountAndTag(accountId, tag); for (VolumeVO volume : volumes) { storage += volume.getSize() == null ? 0L : volume.getSize(); } - return storage; + return snapshotsPhysicalSizeOnPrimaryStorage + storage; } @Override @@ -2143,7 +2144,6 @@ public class ResourceLimitManagerImpl extends ManagerBase implements ResourceLim protected class ResourceCountCheckTask extends ManagedContextRunnable { public ResourceCountCheckTask() { - } @Override diff --git a/server/src/main/java/com/cloud/server/StatsCollector.java b/server/src/main/java/com/cloud/server/StatsCollector.java index 7e83d452bb9..1e0138f7cf9 100644 --- a/server/src/main/java/com/cloud/server/StatsCollector.java +++ b/server/src/main/java/com/cloud/server/StatsCollector.java @@ -1646,7 +1646,7 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc List pools = _storagePoolDao.listAll(); for (StoragePoolVO pool : pools) { - List volumes = _volsDao.findByPoolId(pool.getId(), null); + List volumes = _volsDao.findNonDestroyedVolumesByPoolId(pool.getId(), null); for (VolumeVO volume : volumes) { if (!List.of(ImageFormat.QCOW2, ImageFormat.VHD, ImageFormat.OVA, ImageFormat.RAW).contains(volume.getFormat()) && !List.of(Storage.StoragePoolType.PowerFlex, Storage.StoragePoolType.FiberChannel).contains(pool.getPoolType())) { diff --git a/server/src/main/java/com/cloud/storage/ImageStoreDetailsUtil.java b/server/src/main/java/com/cloud/storage/ImageStoreDetailsUtil.java index baf5ef8902d..9f5aa660f4f 100755 --- a/server/src/main/java/com/cloud/storage/ImageStoreDetailsUtil.java +++ b/server/src/main/java/com/cloud/storage/ImageStoreDetailsUtil.java @@ -78,4 +78,15 @@ public class ImageStoreDetailsUtil { return getGlobalDefaultNfsVersion(); } + public boolean isCopyTemplatesFromOtherStoragesEnabled(Long storeId, Long zoneId) { + final Map storeDetails = imageStoreDetailsDao.getDetails(storeId); + final String keyWithoutDots = StorageManager.COPY_TEMPLATES_FROM_OTHER_SECONDARY_STORAGES.key() + .replace(".", ""); + + if (storeDetails != null && storeDetails.containsKey(keyWithoutDots)) { + return Boolean.parseBoolean(storeDetails.get(keyWithoutDots)); + } + + return StorageManager.COPY_TEMPLATES_FROM_OTHER_SECONDARY_STORAGES.valueIn(zoneId); + } } diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index d23856552ee..d1dca0fa901 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -1558,14 +1558,22 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C protected String getStoragePoolNonDestroyedVolumesLog(long storagePoolId) { StringBuilder sb = new StringBuilder(); - List nonDestroyedVols = volumeDao.findByPoolId(storagePoolId, null).stream().filter(vol -> vol.getState() != Volume.State.Destroy).collect(Collectors.toList()); + List nonDestroyedVols = volumeDao.findNonDestroyedVolumesByPoolId(storagePoolId, null); VMInstanceVO volInstance; List logMessageInfo = new ArrayList<>(); sb.append("["); for (VolumeVO vol : nonDestroyedVols) { - volInstance = _vmInstanceDao.findById(vol.getInstanceId()); - logMessageInfo.add(String.format("Volume [%s] (attached to VM [%s])", vol.getUuid(), volInstance.getUuid())); + if (vol.getInstanceId() != null) { + volInstance = _vmInstanceDao.findById(vol.getInstanceId()); + if (volInstance != null) { + logMessageInfo.add(String.format("Volume [%s] (attached to VM [%s])", vol.getUuid(), volInstance.getUuid())); + } else { + logMessageInfo.add(String.format("Volume [%s] (attached VM with ID [%d] doesn't exists)", vol.getUuid(), vol.getInstanceId())); + } + } else { + logMessageInfo.add(String.format("Volume [%s] (not attached to any VM)", vol.getUuid())); + } } sb.append(String.join(", ", logMessageInfo)); sb.append("]"); @@ -1867,41 +1875,7 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C } } - //destroy snapshots in destroying state in snapshot_store_ref - List ssSnapshots = _snapshotStoreDao.listByState(ObjectInDataStoreStateMachine.State.Destroying); - for (SnapshotDataStoreVO snapshotDataStoreVO : ssSnapshots) { - String snapshotUuid = null; - SnapshotVO snapshot = null; - final String storeRole = snapshotDataStoreVO.getRole().toString().toLowerCase(); - if (logger.isDebugEnabled()) { - snapshot = _snapshotDao.findById(snapshotDataStoreVO.getSnapshotId()); - if (snapshot == null) { - logger.warn(String.format("Did not find snapshot [ID: %d] for which store reference is in destroying state; therefore, it cannot be destroyed.", snapshotDataStoreVO.getSnapshotId())); - continue; - } - snapshotUuid = snapshot.getUuid(); - } - - try { - if (logger.isDebugEnabled()) { - logger.debug(String.format("Verifying if snapshot [%s] is in destroying state in %s data store ID: %d.", snapshotUuid, storeRole, snapshotDataStoreVO.getDataStoreId())); - } - SnapshotInfo snapshotInfo = snapshotFactory.getSnapshot(snapshotDataStoreVO.getSnapshotId(), snapshotDataStoreVO.getDataStoreId(), snapshotDataStoreVO.getRole()); - if (snapshotInfo != null) { - if (logger.isDebugEnabled()) { - logger.debug(String.format("Snapshot [%s] in destroying state found in %s data store [%s]; therefore, it will be destroyed.", snapshotUuid, storeRole, snapshotInfo.getDataStore().getUuid())); - } - _snapshotService.deleteSnapshot(snapshotInfo); - } else if (logger.isDebugEnabled()) { - logger.debug(String.format("Did not find snapshot [%s] in destroying state in %s data store ID: %d.", snapshotUuid, storeRole, snapshotDataStoreVO.getDataStoreId())); - } - } catch (Exception e) { - logger.error("Failed to delete snapshot [{}] from storage due to: [{}].", snapshot, e.getMessage()); - if (logger.isDebugEnabled()) { - logger.debug("Failed to delete snapshot [{}] from storage.", snapshot, e); - } - } - } + cleanupSnapshotsFromStoreRefInDestroyingState(); cleanupSecondaryStorage(recurring); List vols = volumeDao.listVolumesToBeDestroyed(new Date(System.currentTimeMillis() - ((long)StorageCleanupDelay.value() << 10))); @@ -1941,20 +1915,7 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C } } - // remove snapshots in Error state - List snapshots = _snapshotDao.listAllByStatus(Snapshot.State.Error); - for (SnapshotVO snapshotVO : snapshots) { - try { - List storeRefs = _snapshotStoreDao.findBySnapshotId(snapshotVO.getId()); - for (SnapshotDataStoreVO ref : storeRefs) { - _snapshotStoreDao.expunge(ref.getId()); - } - _snapshotDao.expunge(snapshotVO.getId()); - } catch (Exception e) { - logger.error("Unable to destroy snapshot [{}] due to: [{}].", snapshotVO, e.getMessage()); - logger.debug("Unable to destroy snapshot [{}].", snapshotVO, e); - } - } + removeSnapshotsInErrorStatus(); // destroy uploaded volumes in abandoned/error state List volumeDataStores = _volumeDataStoreDao.listByVolumeState(Volume.State.UploadError, Volume.State.UploadAbandoned); @@ -2055,6 +2016,56 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C } } + private void cleanupSnapshotsFromStoreRefInDestroyingState() { + List storeRefSnapshotsInDestroyingState = _snapshotStoreDao.listByState(ObjectInDataStoreStateMachine.State.Destroying); + for (SnapshotDataStoreVO snapshotDataStoreVO : storeRefSnapshotsInDestroyingState) { + SnapshotVO snapshot = _snapshotDao.findById(snapshotDataStoreVO.getSnapshotId()); + if (snapshot == null) { + logger.warn("Did not find snapshot [ID: {}] for which store reference is in destroying state; therefore, it cannot be destroyed.", snapshotDataStoreVO.getSnapshotId()); + continue; + } + deleteSnapshot(snapshot, snapshotDataStoreVO); + } + } + + private void deleteSnapshot(SnapshotVO snapshot, SnapshotDataStoreVO snapshotDataStoreVO) { + if (snapshot == null || snapshotDataStoreVO == null) { + return; + } + + try { + final String snapshotUuid = snapshot.getUuid(); + final String storeRole = snapshotDataStoreVO.getRole().toString().toLowerCase(); + logger.debug("Snapshot [{}] is in {} state on {} data store ID: {}.", snapshotUuid, snapshotDataStoreVO.getState(), storeRole, snapshotDataStoreVO.getDataStoreId()); + SnapshotInfo snapshotInfo = snapshotFactory.getSnapshotIncludingRemoved(snapshotDataStoreVO.getSnapshotId(), snapshotDataStoreVO.getDataStoreId(), snapshotDataStoreVO.getRole()); + if (snapshotInfo != null) { + logger.debug("Snapshot [{}] in {} state found on {} data store [{}], it will be deleted.", snapshotUuid, snapshotDataStoreVO.getState(), storeRole, snapshotInfo.getDataStore().getUuid()); + _snapshotService.deleteSnapshot(snapshotInfo); + } else { + logger.debug("Did not find snapshot [{}] in {} state on {} data store ID: {}.", snapshotUuid, snapshotDataStoreVO.getState(), storeRole, snapshotDataStoreVO.getDataStoreId()); + } + } catch (Exception e) { + logger.error("Failed to delete snapshot [{}] from storage due to: [{}].", snapshot, e.getMessage(), e); + } + } + + private void removeSnapshotsInErrorStatus() { + List snapshotsInErrorStatus = _snapshotDao.listAllByStatusIncludingRemoved(Snapshot.State.Error); + for (SnapshotVO snapshotVO : snapshotsInErrorStatus) { + try { + List storeRefSnapshotsInErrorStatus = _snapshotStoreDao.findBySnapshotId(snapshotVO.getId()); + for (SnapshotDataStoreVO snapshotDataStoreVO : storeRefSnapshotsInErrorStatus) { + deleteSnapshot(snapshotVO, snapshotDataStoreVO); + _snapshotStoreDao.expunge(snapshotDataStoreVO.getId()); + } + _snapshotDao.expunge(snapshotVO.getId()); + } catch (Exception e) { + logger.error("Unable to destroy snapshot [{}] due to: [{}].", snapshotVO, e.getMessage()); + logger.debug("Unable to destroy snapshot [{}].", snapshotVO, e); + } + } + } + protected boolean isVolumeSuspectedDestroyDuplicateOfVmVolume(VolumeVO gcVolume) { if (gcVolume.getPath() == null) { return false; @@ -2633,7 +2644,7 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C for (String childDatastoreUUID : childDatastoreUUIDs) { StoragePoolVO dataStoreVO = _storagePoolDao.findPoolByUUID(childDatastoreUUID); - List allVolumes = volumeDao.findByPoolId(dataStoreVO.getId()); + List allVolumes = volumeDao.findNonDestroyedVolumesByPoolId(dataStoreVO.getId()); allVolumes.removeIf(volumeVO -> volumeVO.getInstanceId() == null); allVolumes.removeIf(volumeVO -> volumeVO.getState() != Volume.State.Ready); for (VolumeVO volume : allVolumes) { @@ -4195,7 +4206,7 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C DataStoreDownloadFollowRedirects, AllowVolumeReSizeBeyondAllocation, StoragePoolHostConnectWorkers, - COPY_PUBLIC_TEMPLATES_FROM_OTHER_STORAGES + COPY_TEMPLATES_FROM_OTHER_SECONDARY_STORAGES }; } diff --git a/server/src/main/java/com/cloud/storage/StoragePoolAutomationImpl.java b/server/src/main/java/com/cloud/storage/StoragePoolAutomationImpl.java index 612582640f4..667af5a876f 100644 --- a/server/src/main/java/com/cloud/storage/StoragePoolAutomationImpl.java +++ b/server/src/main/java/com/cloud/storage/StoragePoolAutomationImpl.java @@ -91,7 +91,7 @@ public class StoragePoolAutomationImpl implements StoragePoolAutomation { boolean restart = !CollectionUtils.isEmpty(upPools); // 2. Get a list of all the ROOT volumes within this storage pool - List allVolumes = volumeDao.findByPoolId(pool.getId()); + List allVolumes = volumeDao.findNonDestroyedVolumesByPoolId(pool.getId()); // 3. Enqueue to the work queue enqueueMigrationsForVolumes(allVolumes, pool); // 4. Process the queue diff --git a/server/src/main/java/com/cloud/storage/listener/StoragePoolMonitor.java b/server/src/main/java/com/cloud/storage/listener/StoragePoolMonitor.java index a0e10c646b5..6df3cbaeedf 100644 --- a/server/src/main/java/com/cloud/storage/listener/StoragePoolMonitor.java +++ b/server/src/main/java/com/cloud/storage/listener/StoragePoolMonitor.java @@ -21,6 +21,7 @@ import java.util.List; import javax.inject.Inject; import com.cloud.storage.StorageManager; +import com.cloud.utils.Profiler; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProvider; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProviderManager; import org.apache.cloudstack.engine.subsystem.api.storage.HypervisorHostListener; @@ -144,12 +145,13 @@ public class StoragePoolMonitor implements Listener { } @Override - public synchronized boolean processDisconnect(long agentId, Status state) { + public boolean processDisconnect(long agentId, Status state) { return processDisconnect(agentId, null, null, state); } @Override - public synchronized boolean processDisconnect(long agentId, String uuid, String name, Status state) { + public boolean processDisconnect(long agentId, String uuid, String name, Status state) { + logger.debug("Starting disconnect for Agent [id: {}, uuid: {}, name: {}]", agentId, uuid, name); Host host = _storageManager.getHost(agentId); if (host == null) { logger.warn("Agent [id: {}, uuid: {}, name: {}] not found, not disconnecting pools", agentId, uuid, name); @@ -157,38 +159,52 @@ public class StoragePoolMonitor implements Listener { } if (host.getType() != Host.Type.Routing) { + logger.debug("Host [id: {}, uuid: {}, name: {}] is not of type {}, skip", agentId, uuid, name, Host.Type.Routing); return false; } + logger.debug("Looking for connected Storage Pools for Host [id: {}, uuid: {}, name: {}]", agentId, uuid, name); List storagePoolHosts = _storageManager.findStoragePoolsConnectedToHost(host.getId()); if (storagePoolHosts == null) { - if (logger.isTraceEnabled()) { - logger.trace("No pools to disconnect for host: {}", host); - } + logger.debug("No pools to disconnect for host: {}", host); return true; } + logger.debug("Found {} pools to disconnect for host: {}", storagePoolHosts.size(), host); boolean disconnectResult = true; - for (StoragePoolHostVO storagePoolHost : storagePoolHosts) { + int storagePoolHostsSize = storagePoolHosts.size(); + for (int i = 0; i < storagePoolHostsSize; i++) { + StoragePoolHostVO storagePoolHost = storagePoolHosts.get(i); + logger.debug("Processing disconnect from Storage Pool {} ({} of {}) for host: {}", storagePoolHost.getPoolId(), i, storagePoolHostsSize, host); StoragePoolVO pool = _poolDao.findById(storagePoolHost.getPoolId()); if (pool == null) { + logger.debug("No Storage Pool found with id {} ({} of {}) for host: {}", storagePoolHost.getPoolId(), i, storagePoolHostsSize, host); continue; } if (!pool.isShared()) { + logger.debug("Storage Pool {} ({}) ({} of {}) is not shared for host: {}, ignore disconnect", pool.getName(), pool.getUuid(), i, storagePoolHostsSize, host); continue; } // Handle only PowerFlex pool for now, not to impact other pools behavior if (pool.getPoolType() != StoragePoolType.PowerFlex) { + logger.debug("Storage Pool {} ({}) ({} of {}) is not of type {} for host: {}, ignore disconnect", pool.getName(), pool.getUuid(), i, storagePoolHostsSize, pool.getPoolType(), host); continue; } + logger.debug("Sending disconnect to Storage Pool {} ({}) ({} of {}) for host: {}", pool.getName(), pool.getUuid(), i, storagePoolHostsSize, host); + Profiler disconnectProfiler = new Profiler(); try { + disconnectProfiler.start(); _storageManager.disconnectHostFromSharedPool(host, pool); } catch (Exception e) { logger.error("Unable to disconnect host {} from storage pool {} due to {}", host, pool, e.toString()); disconnectResult = false; + } finally { + disconnectProfiler.stop(); + long disconnectDuration = disconnectProfiler.getDurationInMillis() / 1000; + logger.debug("Finished disconnect with result {} from Storage Pool {} ({}) ({} of {}) for host: {}, duration: {} secs", disconnectResult, pool.getName(), pool.getUuid(), i, storagePoolHostsSize, host, disconnectDuration); } } diff --git a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java index b4b4220615e..19cde4da0f1 100755 --- a/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java @@ -276,6 +276,15 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement return !DataCenter.Type.Edge.equals(zone.getType()); } + private ResourceType getStoreResourceType(long dataCenterId, Snapshot.LocationType locationType) { + ResourceType storeResourceType = ResourceType.secondary_storage; + if (!isBackupSnapshotToSecondaryForZone(dataCenterId) || + Snapshot.LocationType.PRIMARY.equals(locationType)) { + storeResourceType = ResourceType.primary_storage; + } + return storeResourceType; + } + @Override public String getConfigComponentName() { return SnapshotManager.class.getSimpleName(); @@ -614,7 +623,7 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement _snapshotDao.update(snapshot.getId(), snapshot); snapshotInfo = this.snapshotFactory.getSnapshot(snapshotId, store); - Long snapshotOwnerId = vm.getAccountId(); + long snapshotOwnerId = vm.getAccountId(); try { SnapshotStrategy snapshotStrategy = _storageStrategyFactory.getSnapshotStrategy(snapshot, SnapshotOperation.BACKUP); @@ -622,7 +631,6 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement throw new CloudRuntimeException(String.format("Unable to find Snapshot strategy to handle Snapshot [%s]", snapshot)); } snapshotInfo = snapshotStrategy.backupSnapshot(snapshotInfo); - } catch (Exception e) { logger.debug("Failed to backup Snapshot from Instance Snapshot", e); _resourceLimitMgr.decrementResourceCount(snapshotOwnerId, ResourceType.snapshot); @@ -721,7 +729,7 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement protected Pair, List> getStoreRefsAndZonesForSnapshotDelete(long snapshotId, Long zoneId) { List snapshotStoreRefs = new ArrayList<>(); - List allSnapshotStoreRefs = _snapshotStoreDao.findBySnapshotId(snapshotId); + List allSnapshotStoreRefs = _snapshotStoreDao.findBySnapshotIdWithNonDestroyedState(snapshotId); List zoneIds = new ArrayList<>(); if (zoneId != null) { DataCenterVO zone = dataCenterDao.findById(zoneId); @@ -771,12 +779,11 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement _accountMgr.checkAccess(caller, null, true, snapshotCheck); SnapshotStrategy snapshotStrategy = _storageStrategyFactory.getSnapshotStrategy(snapshotCheck, zoneId, SnapshotOperation.DELETE); - if (snapshotStrategy == null) { logger.error("Unable to find snapshot strategy to handle snapshot [{}]", snapshotCheck); - return false; } + Pair, List> storeRefAndZones = getStoreRefsAndZonesForSnapshotDelete(snapshotId, zoneId); List snapshotStoreRefs = storeRefAndZones.first(); List zoneIds = storeRefAndZones.second(); @@ -1472,8 +1479,9 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement UsageEventUtils.publishUsageEvent(EventTypes.EVENT_SNAPSHOT_CREATE, snapshot.getAccountId(), snapshot.getDataCenterId(), snapshotId, snapshot.getName(), null, null, snapshotStoreRef.getPhysicalSize(), volume.getSize(), snapshot.getClass().getName(), snapshot.getUuid()); + ResourceType storeResourceType = dataStoreRole == DataStoreRole.Image ? ResourceType.secondary_storage : ResourceType.primary_storage; // Correct the resource count of snapshot in case of delta snapshots. - _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), ResourceType.secondary_storage, new Long(volume.getSize() - snapshotStoreRef.getPhysicalSize())); + _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), storeResourceType, new Long(volume.getSize() - snapshotStoreRef.getPhysicalSize())); if (!payload.getAsyncBackup() && backupSnapToSecondary) { copyNewSnapshotToZones(snapshotId, snapshot.getDataCenterId(), payload.getZoneIds()); @@ -1485,15 +1493,17 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement if (logger.isDebugEnabled()) { logger.debug("Failed to create snapshot" + cre.getLocalizedMessage()); } + ResourceType storeResourceType = getStoreResourceType(volume.getDataCenterId(), payload.getLocationType()); _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), ResourceType.snapshot); - _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), ResourceType.secondary_storage, new Long(volume.getSize())); + _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), storeResourceType, new Long(volume.getSize())); throw cre; } catch (Exception e) { if (logger.isDebugEnabled()) { logger.debug("Failed to create snapshot", e); } + ResourceType storeResourceType = getStoreResourceType(volume.getDataCenterId(), payload.getLocationType()); _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), ResourceType.snapshot); - _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), ResourceType.secondary_storage, new Long(volume.getSize())); + _resourceLimitMgr.decrementResourceCount(snapshotOwner.getId(), storeResourceType, new Long(volume.getSize())); throw new CloudRuntimeException("Failed to create snapshot", e); } return snapshot; @@ -1503,22 +1513,22 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement if (asyncBackup) { backupSnapshotExecutor.schedule(new BackupSnapshotTask(snapshotOnPrimary, snapshotBackupRetries - 1, snapshotStrategy, zoneIds), 0, TimeUnit.SECONDS); } else { - SnapshotInfo backupedSnapshot = snapshotStrategy.backupSnapshot(snapshotOnPrimary); - if (backupedSnapshot != null) { + SnapshotInfo backedUpSnapshot = snapshotStrategy.backupSnapshot(snapshotOnPrimary); + if (backedUpSnapshot != null) { snapshotStrategy.postSnapshotCreation(snapshotOnPrimary); } } } protected class BackupSnapshotTask extends ManagedContextRunnable { - SnapshotInfo snapshot; + SnapshotInfo snapshotOnPrimary; int attempts; SnapshotStrategy snapshotStrategy; List zoneIds; - public BackupSnapshotTask(SnapshotInfo snap, int maxRetries, SnapshotStrategy strategy, List zoneIds) { - snapshot = snap; + public BackupSnapshotTask(SnapshotInfo snapshot, int maxRetries, SnapshotStrategy strategy, List zoneIds) { + snapshotOnPrimary = snapshot; attempts = maxRetries; snapshotStrategy = strategy; this.zoneIds = zoneIds; @@ -1529,19 +1539,18 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement try { logger.debug("Value of attempts is " + (snapshotBackupRetries - attempts)); - SnapshotInfo backupedSnapshot = snapshotStrategy.backupSnapshot(snapshot); - - if (backupedSnapshot != null) { - snapshotStrategy.postSnapshotCreation(snapshot); - copyNewSnapshotToZones(snapshot.getId(), snapshot.getDataCenterId(), zoneIds); + SnapshotInfo backedUpSnapshot = snapshotStrategy.backupSnapshot(snapshotOnPrimary); + if (backedUpSnapshot != null) { + snapshotStrategy.postSnapshotCreation(snapshotOnPrimary); + copyNewSnapshotToZones(snapshotOnPrimary.getId(), snapshotOnPrimary.getDataCenterId(), zoneIds); } } catch (final Exception e) { if (attempts >= 0) { - logger.debug("Backing up of snapshot failed, for snapshot {}, left with {} more attempts", snapshot, attempts); - backupSnapshotExecutor.schedule(new BackupSnapshotTask(snapshot, --attempts, snapshotStrategy, zoneIds), snapshotBackupRetryInterval, TimeUnit.SECONDS); + logger.debug("Backing up of snapshot failed, for snapshot {}, left with {} more attempts", snapshotOnPrimary, attempts); + backupSnapshotExecutor.schedule(new BackupSnapshotTask(snapshotOnPrimary, --attempts, snapshotStrategy, zoneIds), snapshotBackupRetryInterval, TimeUnit.SECONDS); } else { - logger.debug("Done with {} attempts in backing up of snapshot {}", snapshotBackupRetries, snapshot.getSnapshotVO()); - snapshotSrv.cleanupOnSnapshotBackupFailure(snapshot); + logger.debug("Done with {} attempts in backing up of snapshot {}", snapshotBackupRetries, snapshotOnPrimary.getSnapshotVO()); + snapshotSrv.cleanupOnSnapshotBackupFailure(snapshotOnPrimary); } } } @@ -1696,11 +1705,7 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement Type snapshotType = getSnapshotType(policyId); Account owner = _accountMgr.getAccount(volume.getAccountId()); - ResourceType storeResourceType = ResourceType.secondary_storage; - if (!isBackupSnapshotToSecondaryForZone(volume.getDataCenterId()) || - Snapshot.LocationType.PRIMARY.equals(locationType)) { - storeResourceType = ResourceType.primary_storage; - } + ResourceType storeResourceType = getStoreResourceType(volume.getDataCenterId(), locationType); try { _resourceLimitMgr.checkResourceLimit(owner, ResourceType.snapshot); _resourceLimitMgr.checkResourceLimit(owner, storeResourceType, volume.getSize()); @@ -1762,7 +1767,7 @@ public class SnapshotManagerImpl extends MutualExclusiveIdsManagerBase implement public void markVolumeSnapshotsAsDestroyed(Volume volume) { List snapshots = _snapshotDao.listByVolumeId(volume.getId()); for (SnapshotVO snapshot: snapshots) { - List snapshotDataStoreVOs = _snapshotStoreDao.findBySnapshotId(snapshot.getId()); + List snapshotDataStoreVOs = _snapshotStoreDao.findBySnapshotIdWithNonDestroyedState(snapshot.getId()); if (CollectionUtils.isEmpty(snapshotDataStoreVOs)) { snapshot.setState(Snapshot.State.Destroyed); _snapshotDao.update(snapshot.getId(), snapshot); diff --git a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java index fb4ea94ae3d..1422e788e24 100644 --- a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java +++ b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java @@ -61,7 +61,6 @@ import org.apache.cloudstack.framework.async.AsyncCompletionCallback; import org.apache.cloudstack.framework.async.AsyncRpcContext; import org.apache.cloudstack.framework.messagebus.MessageBus; import org.apache.cloudstack.framework.messagebus.PublishScope; -import org.apache.cloudstack.secstorage.heuristics.HeuristicType; import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; @@ -308,7 +307,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { for (long zoneId : zonesIds) { - DataStore imageStore = verifyHeuristicRulesForZone(template, zoneId); + DataStore imageStore = templateMgr.verifyHeuristicRulesForZone(template, zoneId); if (imageStore == null) { List imageStores = getImageStoresThrowsExceptionIfNotFound(zoneId, profile); @@ -327,16 +326,6 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { return imageStores; } - protected DataStore verifyHeuristicRulesForZone(VMTemplateVO template, Long zoneId) { - HeuristicType heuristicType; - if (ImageFormat.ISO.equals(template.getFormat())) { - heuristicType = HeuristicType.ISO; - } else { - heuristicType = HeuristicType.TEMPLATE; - } - return heuristicRuleHelper.getImageStoreIfThereIsHeuristicRule(zoneId, heuristicType, template); - } - protected void standardImageStoreAllocation(List imageStores, VMTemplateVO template) { Set zoneSet = new HashSet(); Collections.shuffle(imageStores); @@ -432,7 +421,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { } Long zoneId = zoneIdList.get(0); - DataStore imageStore = verifyHeuristicRulesForZone(template, zoneId); + DataStore imageStore = templateMgr.verifyHeuristicRulesForZone(template, zoneId); List payloads = new LinkedList<>(); if (imageStore == null) { diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index ba8e5714180..78265021c0a 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -122,6 +122,7 @@ import com.cloud.agent.api.to.DatadiskTO; import com.cloud.agent.api.to.DiskTO; import com.cloud.agent.api.to.NfsTO; import com.cloud.agent.api.to.VirtualMachineTO; +import com.cloud.agent.api.to.deployasis.OVFNetworkTO; import com.cloud.api.ApiDBUtils; import com.cloud.api.query.dao.UserVmJoinDao; import com.cloud.api.query.vo.UserVmJoinVO; @@ -131,6 +132,7 @@ import com.cloud.dc.DataCenter; import com.cloud.dc.DataCenterVO; import com.cloud.dc.dao.DataCenterDao; import com.cloud.deploy.DeployDestination; +import com.cloud.deployasis.dao.TemplateDeployAsIsDetailsDao; import com.cloud.domain.Domain; import com.cloud.domain.dao.DomainDao; import com.cloud.event.ActionEvent; @@ -313,6 +315,8 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, protected SnapshotHelper snapshotHelper; @Inject VnfTemplateManager vnfTemplateManager; + @Inject + TemplateDeployAsIsDetailsDao templateDeployAsIsDetailsDao; @Inject private SecondaryStorageHeuristicDao secondaryStorageHeuristicDao; @@ -838,6 +842,9 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, // Copy will just find one eligible image store for the destination zone // and copy template there, not propagate to all image stores // for that zone + + boolean copied = false; + for (DataStore dstSecStore : dstSecStores) { TemplateDataStoreVO dstTmpltStore = _tmplStoreDao.findByStoreTemplate(dstSecStore.getId(), tmpltId); if (dstTmpltStore != null && dstTmpltStore.getDownloadState() == Status.DOWNLOADED) { @@ -852,9 +859,12 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, TemplateApiResult result = future.get(); if (result.isFailed()) { logger.debug("Copy Template failed for image store {}: {}", dstSecStore, result.getResult()); + _tmplStoreDao.removeByTemplateStore(tmpltId, dstSecStore.getId()); continue; // try next image store } + copied = true; + _tmpltDao.addTemplateToZone(template, dstZoneId); if (account.getId() != Account.ACCOUNT_ID_SYSTEM) { @@ -882,12 +892,14 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, } } } + + return true; + } catch (Exception ex) { - logger.debug("Failed to copy Template to image store:{} ,will try next one", dstSecStore); + logger.debug("Failed to copy Template to image store:{} ,will try next one", dstSecStore, ex); } } - return true; - + return copied; } @Override @@ -2172,6 +2184,11 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, templateType = validateTemplateType(cmd, isAdmin, template.isCrossZones()); if (cmd instanceof UpdateVnfTemplateCmd) { VnfTemplateUtils.validateApiCommandParams(cmd, template); + UpdateVnfTemplateCmd updateCmd = (UpdateVnfTemplateCmd) cmd; + if (template.isDeployAsIs() && CollectionUtils.isNotEmpty(updateCmd.getVnfNics())) { + List ovfNetworks = templateDeployAsIsDetailsDao.listNetworkRequirementsByTemplateId(template.getId()); + VnfTemplateUtils.validateDeployAsIsTemplateVnfNics(ovfNetworks, updateCmd.getVnfNics()); + } vnfTemplateManager.updateVnfTemplate(template.getId(), (UpdateVnfTemplateCmd) cmd); } templateTag = ((UpdateTemplateCmd)cmd).getTemplateTag(); @@ -2347,6 +2364,17 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, return templateType; } + @Override + public DataStore verifyHeuristicRulesForZone(VMTemplateVO template, Long zoneId) { + HeuristicType heuristicType; + if (ImageFormat.ISO.equals(template.getFormat())) { + heuristicType = HeuristicType.ISO; + } else { + heuristicType = HeuristicType.TEMPLATE; + } + return heuristicRuleHelper.getImageStoreIfThereIsHeuristicRule(zoneId, heuristicType, template); + } + void validateDetails(VMTemplateVO template, Map details) { if (MapUtils.isEmpty(details)) { return; diff --git a/server/src/main/java/com/cloud/usage/UsageServiceImpl.java b/server/src/main/java/com/cloud/usage/UsageServiceImpl.java index c46f1f43fd0..de8d4633d22 100644 --- a/server/src/main/java/com/cloud/usage/UsageServiceImpl.java +++ b/server/src/main/java/com/cloud/usage/UsageServiceImpl.java @@ -17,7 +17,6 @@ package com.cloud.usage; import java.util.ArrayList; -import java.util.Calendar; import java.util.Date; import java.util.List; import java.util.Map; @@ -26,6 +25,7 @@ import java.util.TimeZone; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.configuration.ConfigurationManagerImpl; import org.apache.cloudstack.api.command.admin.usage.GenerateUsageRecordsCmd; import org.apache.cloudstack.api.command.admin.usage.ListUsageRecordsCmd; import org.apache.cloudstack.api.command.admin.usage.RemoveRawUsageRecordsCmd; @@ -34,6 +34,7 @@ import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.usage.Usage; import org.apache.cloudstack.usage.UsageService; import org.apache.cloudstack.usage.UsageTypes; +import org.apache.cloudstack.utils.usage.UsageUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.jetbrains.annotations.NotNull; @@ -126,14 +127,25 @@ public class UsageServiceImpl extends ManagerBase implements UsageService, Manag @Inject private NetworkOfferingDao _networkOfferingDao; + private TimeZone usageExecutionTimeZone = TimeZone.getTimeZone("GMT"); + + private static final long REMOVE_RAW_USAGE_RECORDS_WINDOW_IN_MS = 15 * 60 * 1000; + public UsageServiceImpl() { } @Override public boolean configure(String name, Map params) throws ConfigurationException { super.configure(name, params); + String timeZoneStr = ObjectUtils.defaultIfNull(_configDao.getValue(Config.UsageAggregationTimezone.toString()), "GMT"); _usageTimezone = TimeZone.getTimeZone(timeZoneStr); + + String executionTimeZone = _configDao.getValue(Config.UsageExecutionTimezone.toString()); + if (executionTimeZone != null) { + usageExecutionTimeZone = TimeZone.getTimeZone(executionTimeZone); + } + return true; } @@ -464,35 +476,28 @@ public class UsageServiceImpl extends ManagerBase implements UsageService, Manag @Override public boolean removeRawUsageRecords(RemoveRawUsageRecordsCmd cmd) throws InvalidParameterValueException { Integer interval = cmd.getInterval(); - if (interval != null && interval > 0 ) { - String jobExecTime = _configDao.getValue(Config.UsageStatsJobExecTime.toString()); - if (jobExecTime != null ) { - String[] segments = jobExecTime.split(":"); - if (segments.length == 2) { - String timeZoneStr = _configDao.getValue(Config.UsageExecutionTimezone.toString()); - if (timeZoneStr == null) { - timeZoneStr = "GMT"; - } - TimeZone tz = TimeZone.getTimeZone(timeZoneStr); - Calendar cal = Calendar.getInstance(tz); - cal.setTime(new Date()); - long curTS = cal.getTimeInMillis(); - cal.set(Calendar.HOUR_OF_DAY, Integer.parseInt(segments[0])); - cal.set(Calendar.MINUTE, Integer.parseInt(segments[1])); - cal.set(Calendar.SECOND, 0); - cal.set(Calendar.MILLISECOND, 0); - long execTS = cal.getTimeInMillis(); - logger.debug("Trying to remove old raw cloud_usage records older than " + interval + " day(s), current time=" + curTS + " next job execution time=" + execTS); - // Let's avoid cleanup when job runs and around a 15 min interval - if (Math.abs(curTS - execTS) < 15 * 60 * 1000) { - return false; - } - } - } - _usageDao.removeOldUsageRecords(interval); - } else { - throw new InvalidParameterValueException("Invalid interval value. Interval to remove cloud_usage records should be greater than 0"); + if (interval == null || interval <= 0) { + throw new InvalidParameterValueException("Interval should be greater than 0."); } + + String jobExecTime = _configDao.getValue(Config.UsageStatsJobExecTime.toString()); + Date previousJobExecTime = UsageUtils.getPreviousJobExecutionTime(usageExecutionTimeZone, jobExecTime); + Date nextJobExecTime = UsageUtils.getNextJobExecutionTime(usageExecutionTimeZone, jobExecTime); + if (ObjectUtils.allNotNull(previousJobExecTime, nextJobExecTime)) { + logger.debug("Next Usage job is scheduled to execute at [{}]; previous execution was at [{}].", + DateUtil.displayDateInTimezone(usageExecutionTimeZone, nextJobExecTime), DateUtil.displayDateInTimezone(usageExecutionTimeZone, previousJobExecTime)); + Date now = new Date(); + if (nextJobExecTime.getTime() - now.getTime() < REMOVE_RAW_USAGE_RECORDS_WINDOW_IN_MS) { + logger.info("Not removing any cloud_usage records because the next Usage job is scheduled to execute in less than {} minute(s).", REMOVE_RAW_USAGE_RECORDS_WINDOW_IN_MS / 60000); + return false; + } else if (now.getTime() - previousJobExecTime.getTime() < REMOVE_RAW_USAGE_RECORDS_WINDOW_IN_MS) { + logger.info("Not removing any cloud_usage records because the last Usage job executed in less than {} minute(s) ago.", REMOVE_RAW_USAGE_RECORDS_WINDOW_IN_MS / 60000); + return false; + } + } + + logger.info("Removing cloud_usage records older than {} day(s).", interval); + _usageDao.expungeAllOlderThan(interval, ConfigurationManagerImpl.DELETE_QUERY_BATCH_SIZE.value()); return true; } } diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 0e407d8b628..bfaaa5cbd19 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -2261,7 +2261,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir private List getVolumesByHost(HostVO host, StoragePool pool){ List vmsPerHost = _vmInstanceDao.listByHostId(host.getId()); return vmsPerHost.stream() - .flatMap(vm -> _volsDao.findByInstanceIdAndPoolId(vm.getId(),pool.getId()).stream().map(vol -> + .flatMap(vm -> _volsDao.findNonDestroyedVolumesByInstanceIdAndPoolId(vm.getId(),pool.getId()).stream().map(vol -> vol.getState() == Volume.State.Ready ? (vol.getFormat() == ImageFormat.OVA ? vol.getChainInfo() : vol.getPath()) : null).filter(Objects::nonNull)) .collect(Collectors.toList()); } @@ -6127,7 +6127,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir throw new InvalidParameterValueException("Unable to use template " + templateId); } if (TemplateType.VNF.equals(template.getTemplateType())) { - vnfTemplateManager.validateVnfApplianceNics(template, cmd.getNetworkIds()); + vnfTemplateManager.validateVnfApplianceNics(template, cmd.getNetworkIds(), cmd.getVmNetworkMap()); } else if (cmd instanceof DeployVnfApplianceCmd) { throw new InvalidParameterValueException("Can't deploy VNF appliance from a non-VNF template"); } diff --git a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java index 1e6ef1a7852..dc2677a507f 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java @@ -239,7 +239,8 @@ public class BackupManagerImpl extends ManagerBase implements BackupManager { final Filter searchFilter = new Filter(BackupOfferingVO.class, "id", true, cmd.getStartIndex(), cmd.getPageSizeVal()); SearchBuilder sb = backupOfferingDao.createSearchBuilder(); sb.and("zone_id", sb.entity().getZoneId(), SearchCriteria.Op.EQ); - sb.and("name", sb.entity().getName(), SearchCriteria.Op.EQ); + sb.and("name", sb.entity().getName(), SearchCriteria.Op.LIKE); + CallContext ctx = CallContext.current(); final Account caller = ctx.getCallingAccount(); if (Account.Type.NORMAL == caller.getType()) { diff --git a/server/src/main/java/org/apache/cloudstack/storage/heuristics/HeuristicRuleHelper.java b/server/src/main/java/org/apache/cloudstack/storage/heuristics/HeuristicRuleHelper.java index 21a34de0d23..2e0780e7fe8 100644 --- a/server/src/main/java/org/apache/cloudstack/storage/heuristics/HeuristicRuleHelper.java +++ b/server/src/main/java/org/apache/cloudstack/storage/heuristics/HeuristicRuleHelper.java @@ -117,8 +117,8 @@ public class HeuristicRuleHelper { accountId = ((SnapshotInfo) obj).getAccountId(); break; case VOLUME: - presetVariables.setVolume(setVolumePresetVariable((VolumeVO) obj)); - accountId = ((VolumeVO) obj).getAccountId(); + presetVariables.setVolume(setVolumePresetVariable((com.cloud.storage.Volume) obj)); + accountId = ((com.cloud.storage.Volume) obj).getAccountId(); break; } presetVariables.setAccount(setAccountPresetVariable(accountId)); @@ -191,14 +191,14 @@ public class HeuristicRuleHelper { return template; } - protected Volume setVolumePresetVariable(VolumeVO volumeVO) { - Volume volume = new Volume(); + protected Volume setVolumePresetVariable(com.cloud.storage.Volume volumeVO) { + Volume volumePresetVariable = new Volume(); - volume.setName(volumeVO.getName()); - volume.setFormat(volumeVO.getFormat()); - volume.setSize(volumeVO.getSize()); + volumePresetVariable.setName(volumeVO.getName()); + volumePresetVariable.setFormat(volumeVO.getFormat()); + volumePresetVariable.setSize(volumeVO.getSize()); - return volume; + return volumePresetVariable; } protected Snapshot setSnapshotPresetVariable(SnapshotInfo snapshotInfo) { diff --git a/server/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateManagerImpl.java b/server/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateManagerImpl.java index ef0f6f6b226..0ebff237a44 100644 --- a/server/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/storage/template/VnfTemplateManagerImpl.java @@ -201,7 +201,14 @@ public class VnfTemplateManagerImpl extends ManagerBase implements VnfTemplateMa } @Override - public void validateVnfApplianceNics(VirtualMachineTemplate template, List networkIds) { + public void validateVnfApplianceNics(VirtualMachineTemplate template, List networkIds, Map vmNetworkMap) { + if (template.isDeployAsIs()) { + if (CollectionUtils.isNotEmpty(networkIds)) { + throw new InvalidParameterValueException("VNF nics mappings should be empty for deploy-as-is templates"); + } + validateVnfApplianceNetworksMap(template, vmNetworkMap); + return; + } if (CollectionUtils.isEmpty(networkIds)) { throw new InvalidParameterValueException("VNF nics list is empty"); } @@ -213,6 +220,18 @@ public class VnfTemplateManagerImpl extends ManagerBase implements VnfTemplateMa } } + private void validateVnfApplianceNetworksMap(VirtualMachineTemplate template, Map vmNetworkMap) { + if (MapUtils.isEmpty(vmNetworkMap)) { + throw new InvalidParameterValueException("VNF networks map is empty"); + } + List vnfNics = vnfTemplateNicDao.listByTemplateId(template.getId()); + for (VnfTemplateNicVO vnfNic : vnfNics) { + if (vnfNic.isRequired() && vmNetworkMap.size() <= vnfNic.getDeviceId()) { + throw new InvalidParameterValueException("VNF nic is required but not found: " + vnfNic); + } + } + } + protected Set getOpenPortsForVnfAppliance(VirtualMachineTemplate template) { Set ports = new HashSet<>(); VnfTemplateDetailVO accessMethodsDetail = vnfTemplateDetailsDao.findDetail(template.getId(), VNF.AccessDetail.ACCESS_METHODS.name().toLowerCase()); diff --git a/server/src/main/java/org/apache/cloudstack/vm/schedule/VMSchedulerImpl.java b/server/src/main/java/org/apache/cloudstack/vm/schedule/VMSchedulerImpl.java index 7410fb1c265..56d794fa5c2 100644 --- a/server/src/main/java/org/apache/cloudstack/vm/schedule/VMSchedulerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/vm/schedule/VMSchedulerImpl.java @@ -162,7 +162,13 @@ public class VMSchedulerImpl extends ManagerBase implements VMScheduler, Configu } Date scheduledDateTime = Date.from(ts.toInstant()); - VMScheduledJobVO scheduledJob = new VMScheduledJobVO(vmSchedule.getVmId(), vmSchedule.getId(), vmSchedule.getAction(), scheduledDateTime); + VMScheduledJobVO scheduledJob = vmScheduledJobDao.findByScheduleAndTimestamp(vmSchedule.getId(), scheduledDateTime); + if (scheduledJob != null) { + logger.trace("Job is already scheduled for schedule {} at {}", vmSchedule, scheduledDateTime); + return scheduledDateTime; + } + + scheduledJob = new VMScheduledJobVO(vmSchedule.getVmId(), vmSchedule.getId(), vmSchedule.getAction(), scheduledDateTime); try { vmScheduledJobDao.persist(scheduledJob); ActionEventUtils.onScheduledActionEvent(User.UID_SYSTEM, vm.getAccountId(), actionEventMap.get(vmSchedule.getAction()), diff --git a/server/src/test/java/com/cloud/resource/ResourceManagerImplTest.java b/server/src/test/java/com/cloud/resource/ResourceManagerImplTest.java index 414d41145f7..1669d7a47d9 100644 --- a/server/src/test/java/com/cloud/resource/ResourceManagerImplTest.java +++ b/server/src/test/java/com/cloud/resource/ResourceManagerImplTest.java @@ -45,6 +45,7 @@ import com.cloud.vm.dao.VMInstanceDao; import com.trilead.ssh2.Connection; import org.apache.cloudstack.api.command.admin.host.CancelHostAsDegradedCmd; import org.apache.cloudstack.api.command.admin.host.DeclareHostAsDegradedCmd; +import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.junit.After; import org.junit.Assert; @@ -61,6 +62,7 @@ import org.mockito.MockitoAnnotations; import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; +import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -152,6 +154,12 @@ public class ResourceManagerImplTest { private MockedConstruction getVncPortCommandMockedConstruction; private AutoCloseable closeable; + private void overrideDefaultConfigValue(final ConfigKey configKey, final String name, final Object o) throws IllegalAccessException, NoSuchFieldException { + Field f = ConfigKey.class.getDeclaredField(name); + f.setAccessible(true); + f.set(configKey, o); + } + @Before public void setup() throws Exception { closeable = MockitoAnnotations.openMocks(this); @@ -194,12 +202,12 @@ public class ResourceManagerImplTest { eq("service cloudstack-agent restart"))). willReturn(new SSHCmdHelper.SSHCmdResult(0,"","")); - when(configurationDao.getValue(ResourceManager.KvmSshToAgentEnabled.key())).thenReturn("true"); + overrideDefaultConfigValue(ResourceManager.KvmSshToAgentEnabled, "_defaultValue", "true"); rootDisks = Arrays.asList(rootDisk1, rootDisk2); dataDisks = Collections.singletonList(dataDisk); - when(volumeDao.findByPoolId(poolId)).thenReturn(rootDisks); - when(volumeDao.findByPoolId(poolId, Volume.Type.DATADISK)).thenReturn(dataDisks); + when(volumeDao.findNonDestroyedVolumesByPoolId(poolId)).thenReturn(rootDisks); + when(volumeDao.findNonDestroyedVolumesByPoolId(poolId, Volume.Type.DATADISK)).thenReturn(dataDisks); } @After @@ -372,9 +380,9 @@ public class ResourceManagerImplTest { } @Test(expected = CloudRuntimeException.class) - public void testHandleAgentSSHDisabledNotConnectedAgent() { + public void testHandleAgentSSHDisabledNotConnectedAgent() throws NoSuchFieldException, IllegalAccessException { when(host.getStatus()).thenReturn(Status.Disconnected); - when(configurationDao.getValue(ResourceManager.KvmSshToAgentEnabled.key())).thenReturn("false"); + overrideDefaultConfigValue(ResourceManager.KvmSshToAgentEnabled, "_defaultValue", "false"); resourceManager.handleAgentIfNotConnected(host, false); } @@ -564,22 +572,22 @@ public class ResourceManagerImplTest { @Test public void testDestroyLocalStoragePoolVolumesOnlyRootDisks() { - when(volumeDao.findByPoolId(poolId, Volume.Type.DATADISK)).thenReturn(null); + when(volumeDao.findNonDestroyedVolumesByPoolId(poolId, Volume.Type.DATADISK)).thenReturn(null); resourceManager.destroyLocalStoragePoolVolumes(poolId); verify(volumeDao, times(rootDisks.size())).updateAndRemoveVolume(any(VolumeVO.class)); } @Test public void testDestroyLocalStoragePoolVolumesOnlyDataDisks() { - when(volumeDao.findByPoolId(poolId)).thenReturn(null); + when(volumeDao.findNonDestroyedVolumesByPoolId(poolId)).thenReturn(null); resourceManager.destroyLocalStoragePoolVolumes(poolId); verify(volumeDao, times(dataDisks.size())).updateAndRemoveVolume(any(VolumeVO.class)); } @Test public void testDestroyLocalStoragePoolVolumesNoDisks() { - when(volumeDao.findByPoolId(poolId)).thenReturn(null); - when(volumeDao.findByPoolId(poolId, Volume.Type.DATADISK)).thenReturn(null); + when(volumeDao.findNonDestroyedVolumesByPoolId(poolId)).thenReturn(null); + when(volumeDao.findNonDestroyedVolumesByPoolId(poolId, Volume.Type.DATADISK)).thenReturn(null); resourceManager.destroyLocalStoragePoolVolumes(poolId); verify(volumeDao, never()).updateAndRemoveVolume(any(VolumeVO.class)); } diff --git a/server/src/test/java/com/cloud/resourcelimit/ResourceLimitManagerImplTest.java b/server/src/test/java/com/cloud/resourcelimit/ResourceLimitManagerImplTest.java index 34030626d22..53ccc830dd2 100644 --- a/server/src/test/java/com/cloud/resourcelimit/ResourceLimitManagerImplTest.java +++ b/server/src/test/java/com/cloud/resourcelimit/ResourceLimitManagerImplTest.java @@ -28,6 +28,7 @@ import org.apache.cloudstack.api.response.DomainResponse; import org.apache.cloudstack.api.response.TaggedResourceLimitAndCountResponse; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.reservation.dao.ReservationDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; @@ -118,6 +119,8 @@ public class ResourceLimitManagerImplTest extends TestCase { VolumeDao volumeDao; @Mock UserVmDao userVmDao; + @Mock + SnapshotDataStoreDao snapshotDataStoreDao; private List hostTags = List.of("htag1", "htag2", "htag3"); private List storageTags = List.of("stag1", "stag2"); @@ -840,12 +843,13 @@ public class ResourceLimitManagerImplTest extends TestCase { String tag = null; Mockito.when(vmDao.findIdsOfAllocatedVirtualRoutersForAccount(accountId)) .thenReturn(List.of(1L)); + Mockito.when(snapshotDataStoreDao.getSnapshotsPhysicalSizeOnPrimaryStorageByAccountId(accountId)).thenReturn(100L); Mockito.when(volumeDao.primaryStorageUsedForAccount(Mockito.eq(accountId), Mockito.anyList())).thenReturn(100L); - Assert.assertEquals(100L, resourceLimitManager.calculatePrimaryStorageForAccount(accountId, tag)); + Assert.assertEquals(200L, resourceLimitManager.calculatePrimaryStorageForAccount(accountId, tag)); tag = ""; Mockito.when(volumeDao.primaryStorageUsedForAccount(Mockito.eq(accountId), Mockito.anyList())).thenReturn(200L); - Assert.assertEquals(200L, resourceLimitManager.calculatePrimaryStorageForAccount(accountId, tag)); + Assert.assertEquals(300L, resourceLimitManager.calculatePrimaryStorageForAccount(accountId, tag)); tag = "tag"; VolumeVO vol = Mockito.mock(VolumeVO.class); @@ -853,7 +857,7 @@ public class ResourceLimitManagerImplTest extends TestCase { Mockito.when(vol.getSize()).thenReturn(size); List vols = List.of(vol, vol); Mockito.doReturn(vols).when(resourceLimitManager).getVolumesWithAccountAndTag(accountId, tag); - Assert.assertEquals(vols.size() * size, resourceLimitManager.calculatePrimaryStorageForAccount(accountId, tag)); + Assert.assertEquals((vols.size() * size) + 100L, resourceLimitManager.calculatePrimaryStorageForAccount(accountId, tag)); } @Test diff --git a/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java b/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java index 4a28e044d9c..5f02c89339a 100644 --- a/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java +++ b/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java @@ -531,7 +531,7 @@ public class StorageManagerImplTest { } @Test - public void getStoragePoolNonDestroyedVolumesLogTestNonDestroyedVolumesReturnLog() { + public void getStoragePoolNonDestroyedVolumesLogTestNonDestroyedVolumes_VMAttachedLogs() { Mockito.doReturn(1L).when(storagePoolVOMock).getId(); Mockito.doReturn(1L).when(volume1VOMock).getInstanceId(); Mockito.doReturn("786633d1-a942-4374-9d56-322dd4b0d202").when(volume1VOMock).getUuid(); @@ -539,7 +539,7 @@ public class StorageManagerImplTest { Mockito.doReturn("ffb46333-e983-4c21-b5f0-51c5877a3805").when(volume2VOMock).getUuid(); Mockito.doReturn("58760044-928f-4c4e-9fef-d0e48423595e").when(vmInstanceVOMock).getUuid(); - Mockito.when(_volumeDao.findByPoolId(storagePoolVOMock.getId(), null)).thenReturn(List.of(volume1VOMock, volume2VOMock)); + Mockito.when(_volumeDao.findNonDestroyedVolumesByPoolId(storagePoolVOMock.getId(), null)).thenReturn(List.of(volume1VOMock, volume2VOMock)); Mockito.doReturn(vmInstanceVOMock).when(vmInstanceDao).findById(Mockito.anyLong()); String log = storageManagerImpl.getStoragePoolNonDestroyedVolumesLog(storagePoolVOMock.getId()); @@ -548,6 +548,58 @@ public class StorageManagerImplTest { Assert.assertEquals(expected, log); } + @Test + public void getStoragePoolNonDestroyedVolumesLogTestNonDestroyedVolumes_VMLogForOneVolume() { + Mockito.doReturn(1L).when(storagePoolVOMock).getId(); + Mockito.doReturn(null).when(volume1VOMock).getInstanceId(); + Mockito.doReturn("786633d1-a942-4374-9d56-322dd4b0d202").when(volume1VOMock).getUuid(); + Mockito.doReturn(1L).when(volume2VOMock).getInstanceId(); + Mockito.doReturn("ffb46333-e983-4c21-b5f0-51c5877a3805").when(volume2VOMock).getUuid(); + Mockito.doReturn("58760044-928f-4c4e-9fef-d0e48423595e").when(vmInstanceVOMock).getUuid(); + + Mockito.when(_volumeDao.findNonDestroyedVolumesByPoolId(storagePoolVOMock.getId(), null)).thenReturn(List.of(volume1VOMock, volume2VOMock)); + Mockito.doReturn(vmInstanceVOMock).when(vmInstanceDao).findById(Mockito.anyLong()); + + String log = storageManagerImpl.getStoragePoolNonDestroyedVolumesLog(storagePoolVOMock.getId()); + String expected = String.format("[Volume [%s] (not attached to any VM), Volume [%s] (attached to VM [%s])]", volume1VOMock.getUuid(), volume2VOMock.getUuid(), vmInstanceVOMock.getUuid()); + + Assert.assertEquals(expected, log); + } + + @Test + public void getStoragePoolNonDestroyedVolumesLogTestNonDestroyedVolumes_NotAttachedLogs() { + Mockito.doReturn(1L).when(storagePoolVOMock).getId(); + Mockito.doReturn(null).when(volume1VOMock).getInstanceId(); + Mockito.doReturn("786633d1-a942-4374-9d56-322dd4b0d202").when(volume1VOMock).getUuid(); + Mockito.doReturn(null).when(volume2VOMock).getInstanceId(); + Mockito.doReturn("ffb46333-e983-4c21-b5f0-51c5877a3805").when(volume2VOMock).getUuid(); + + Mockito.when(_volumeDao.findNonDestroyedVolumesByPoolId(storagePoolVOMock.getId(), null)).thenReturn(List.of(volume1VOMock, volume2VOMock)); + + String log = storageManagerImpl.getStoragePoolNonDestroyedVolumesLog(storagePoolVOMock.getId()); + String expected = String.format("[Volume [%s] (not attached to any VM), Volume [%s] (not attached to any VM)]", volume1VOMock.getUuid(), volume2VOMock.getUuid()); + + Assert.assertEquals(expected, log); + } + + @Test + public void getStoragePoolNonDestroyedVolumesLogTestNonDestroyedVolumes_VMNotExistsLog() { + Mockito.doReturn(1L).when(storagePoolVOMock).getId(); + Mockito.doReturn(1L).when(volume1VOMock).getInstanceId(); + Mockito.doReturn("786633d1-a942-4374-9d56-322dd4b0d202").when(volume1VOMock).getUuid(); + Mockito.doReturn(1L).when(volume2VOMock).getInstanceId(); + Mockito.doReturn("ffb46333-e983-4c21-b5f0-51c5877a3805").when(volume2VOMock).getUuid(); + + Mockito.when(_volumeDao.findNonDestroyedVolumesByPoolId(storagePoolVOMock.getId(), null)).thenReturn(List.of(volume1VOMock, volume2VOMock)); + Mockito.doReturn(null).when(vmInstanceDao).findById(Mockito.anyLong()); + + String log = storageManagerImpl.getStoragePoolNonDestroyedVolumesLog(storagePoolVOMock.getId()); + String expected = String.format("[Volume [%s] (attached VM with ID [%d] doesn't exists), Volume [%s] (attached VM with ID [%d] doesn't exists)]", + volume1VOMock.getUuid(), volume1VOMock.getInstanceId(), volume2VOMock.getUuid(), volume2VOMock.getInstanceId()); + + Assert.assertEquals(expected, log); + } + private ChangeStoragePoolScopeCmd mockChangeStoragePooolScopeCmd(String newScope) { ChangeStoragePoolScopeCmd cmd = new ChangeStoragePoolScopeCmd(); ReflectionTestUtils.setField(cmd, "id", 1L); diff --git a/server/src/test/java/com/cloud/storage/snapshot/SnapshotManagerImplTest.java b/server/src/test/java/com/cloud/storage/snapshot/SnapshotManagerImplTest.java index e6c2a0d0f3c..86fdcfecc13 100644 --- a/server/src/test/java/com/cloud/storage/snapshot/SnapshotManagerImplTest.java +++ b/server/src/test/java/com/cloud/storage/snapshot/SnapshotManagerImplTest.java @@ -139,7 +139,7 @@ public class SnapshotManagerImplTest { Mockito.when(ref1.getDataStoreId()).thenReturn(2L); Mockito.when(ref1.getRole()).thenReturn(DataStoreRole.Image); List snapshotStoreList = List.of(ref, ref1); - Mockito.when(snapshotStoreDao.findBySnapshotId(snapshotId)).thenReturn(snapshotStoreList); + Mockito.when(snapshotStoreDao.findBySnapshotIdWithNonDestroyedState(snapshotId)).thenReturn(snapshotStoreList); Mockito.when(dataStoreManager.getStoreZoneId(1L, DataStoreRole.Image)).thenReturn(100L); Mockito.when(dataStoreManager.getStoreZoneId(2L, DataStoreRole.Image)).thenReturn(101L); Pair, List> pair = snapshotManager.getStoreRefsAndZonesForSnapshotDelete(snapshotId, null); @@ -164,7 +164,7 @@ public class SnapshotManagerImplTest { Mockito.when(ref2.getDataStoreId()).thenReturn(3L); Mockito.when(ref2.getRole()).thenReturn(DataStoreRole.Image); List snapshotStoreList = List.of(ref, ref1, ref2); - Mockito.when(snapshotStoreDao.findBySnapshotId(snapshotId)).thenReturn(snapshotStoreList); + Mockito.when(snapshotStoreDao.findBySnapshotIdWithNonDestroyedState(snapshotId)).thenReturn(snapshotStoreList); Mockito.when(dataStoreManager.getStoreZoneId(1L, DataStoreRole.Image)).thenReturn(zoneId); Mockito.when(dataStoreManager.getStoreZoneId(2L, DataStoreRole.Primary)).thenReturn(zoneId); Mockito.when(dataStoreManager.getStoreZoneId(3L, DataStoreRole.Image)).thenReturn(2L); diff --git a/server/src/test/java/com/cloud/template/HypervisorTemplateAdapterTest.java b/server/src/test/java/com/cloud/template/HypervisorTemplateAdapterTest.java index f04adf758c3..82cff280515 100644 --- a/server/src/test/java/com/cloud/template/HypervisorTemplateAdapterTest.java +++ b/server/src/test/java/com/cloud/template/HypervisorTemplateAdapterTest.java @@ -49,7 +49,6 @@ import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.framework.events.Event; import org.apache.cloudstack.framework.events.EventDistributor; import org.apache.cloudstack.framework.messagebus.MessageBus; -import org.apache.cloudstack.secstorage.heuristics.HeuristicType; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import org.apache.cloudstack.storage.heuristics.HeuristicRuleHelper; @@ -341,7 +340,7 @@ public class HypervisorTemplateAdapterTest { Mockito.when(templateProfileMock.getZoneIdList()).thenReturn(zoneIds); Mockito.doReturn(null).when(_adapter).getImageStoresThrowsExceptionIfNotFound(Mockito.any(Long.class), Mockito.any(TemplateProfile.class)); - Mockito.doReturn(null).when(_adapter).verifyHeuristicRulesForZone(Mockito.any(VMTemplateVO.class), Mockito.anyLong()); + Mockito.doReturn(null).when(_templateMgr).verifyHeuristicRulesForZone(Mockito.any(VMTemplateVO.class), Mockito.anyLong()); Mockito.doNothing().when(_adapter).standardImageStoreAllocation(Mockito.isNull(), Mockito.any(VMTemplateVO.class)); _adapter.createTemplateWithinZones(templateProfileMock, vmTemplateVOMock); @@ -357,7 +356,7 @@ public class HypervisorTemplateAdapterTest { Mockito.when(templateProfileMock.getZoneIdList()).thenReturn(zoneIds); Mockito.doReturn(null).when(_adapter).getImageStoresThrowsExceptionIfNotFound(Mockito.any(Long.class), Mockito.any(TemplateProfile.class)); - Mockito.doReturn(null).when(_adapter).verifyHeuristicRulesForZone(Mockito.any(VMTemplateVO.class), Mockito.anyLong()); + Mockito.doReturn(null).when(_templateMgr).verifyHeuristicRulesForZone(Mockito.any(VMTemplateVO.class), Mockito.anyLong()); Mockito.doNothing().when(_adapter).standardImageStoreAllocation(Mockito.isNull(), Mockito.any(VMTemplateVO.class)); _adapter.createTemplateWithinZones(templateProfileMock, vmTemplateVOMock); @@ -373,7 +372,7 @@ public class HypervisorTemplateAdapterTest { List zoneIds = List.of(1L); Mockito.when(templateProfileMock.getZoneIdList()).thenReturn(zoneIds); - Mockito.doReturn(dataStoreMock).when(_adapter).verifyHeuristicRulesForZone(Mockito.any(VMTemplateVO.class), Mockito.anyLong()); + Mockito.doReturn(dataStoreMock).when(_templateMgr).verifyHeuristicRulesForZone(Mockito.any(VMTemplateVO.class), Mockito.anyLong()); Mockito.doNothing().when(_adapter).validateSecondaryStorageAndCreateTemplate(Mockito.any(List.class), Mockito.any(VMTemplateVO.class), Mockito.isNull()); _adapter.createTemplateWithinZones(templateProfileMock, vmTemplateVOMock); @@ -411,26 +410,6 @@ public class HypervisorTemplateAdapterTest { _adapter.getImageStoresThrowsExceptionIfNotFound(zoneId, templateProfileMock); } - @Test - public void verifyHeuristicRulesForZoneTestTemplateIsISOFormatShouldCheckForISOHeuristicType() { - VMTemplateVO vmTemplateVOMock = Mockito.mock(VMTemplateVO.class); - - Mockito.when(vmTemplateVOMock.getFormat()).thenReturn(ImageFormat.ISO); - _adapter.verifyHeuristicRulesForZone(vmTemplateVOMock, 1L); - - Mockito.verify(heuristicRuleHelperMock, Mockito.times(1)).getImageStoreIfThereIsHeuristicRule(1L, HeuristicType.ISO, vmTemplateVOMock); - } - - @Test - public void verifyHeuristicRulesForZoneTestTemplateNotISOFormatShouldCheckForTemplateHeuristicType() { - VMTemplateVO vmTemplateVOMock = Mockito.mock(VMTemplateVO.class); - - Mockito.when(vmTemplateVOMock.getFormat()).thenReturn(ImageFormat.QCOW2); - _adapter.verifyHeuristicRulesForZone(vmTemplateVOMock, 1L); - - Mockito.verify(heuristicRuleHelperMock, Mockito.times(1)).getImageStoreIfThereIsHeuristicRule(1L, HeuristicType.TEMPLATE, vmTemplateVOMock); - } - @Test public void isZoneAndImageStoreAvailableTestZoneIdIsNullShouldReturnFalse() { DataStore dataStoreMock = Mockito.mock(DataStore.class); diff --git a/server/src/test/java/com/cloud/template/TemplateManagerImplTest.java b/server/src/test/java/com/cloud/template/TemplateManagerImplTest.java index d687063f8e3..d4e6d4e80f3 100755 --- a/server/src/test/java/com/cloud/template/TemplateManagerImplTest.java +++ b/server/src/test/java/com/cloud/template/TemplateManagerImplTest.java @@ -23,6 +23,7 @@ import com.cloud.agent.AgentManager; import com.cloud.api.query.dao.UserVmJoinDao; import com.cloud.configuration.Resource; import com.cloud.dc.dao.DataCenterDao; +import com.cloud.deployasis.dao.TemplateDeployAsIsDetailsDao; import com.cloud.domain.dao.DomainDao; import com.cloud.event.dao.UsageEventDao; import com.cloud.exception.InvalidParameterValueException; @@ -205,6 +206,8 @@ public class TemplateManagerImplTest { AccountManager _accountMgr; @Inject VnfTemplateManager vnfTemplateManager; + @Inject + TemplateDeployAsIsDetailsDao templateDeployAsIsDetailsDao; @Inject HeuristicRuleHelper heuristicRuleHelperMock; @@ -752,6 +755,26 @@ public class TemplateManagerImplTest { Assert.assertNull(type); } + @Test + public void verifyHeuristicRulesForZoneTestTemplateIsISOFormatShouldCheckForISOHeuristicType() { + VMTemplateVO vmTemplateVOMock = Mockito.mock(VMTemplateVO.class); + + Mockito.when(vmTemplateVOMock.getFormat()).thenReturn(Storage.ImageFormat.ISO); + templateManager.verifyHeuristicRulesForZone(vmTemplateVOMock, 1L); + + Mockito.verify(heuristicRuleHelperMock, Mockito.times(1)).getImageStoreIfThereIsHeuristicRule(1L, HeuristicType.ISO, vmTemplateVOMock); + } + + @Test + public void verifyHeuristicRulesForZoneTestTemplateNotISOFormatShouldCheckForTemplateHeuristicType() { + VMTemplateVO vmTemplateVOMock = Mockito.mock(VMTemplateVO.class); + + Mockito.when(vmTemplateVOMock.getFormat()).thenReturn(Storage.ImageFormat.QCOW2); + templateManager.verifyHeuristicRulesForZone(vmTemplateVOMock, 1L); + + Mockito.verify(heuristicRuleHelperMock, Mockito.times(1)).getImageStoreIfThereIsHeuristicRule(1L, HeuristicType.TEMPLATE, vmTemplateVOMock); + } + @Configuration @ComponentScan(basePackageClasses = {TemplateManagerImpl.class}, includeFilters = {@ComponentScan.Filter(value = TestConfiguration.Library.class, type = FilterType.CUSTOM)}, @@ -958,6 +981,11 @@ public class TemplateManagerImplTest { return Mockito.mock(VnfTemplateManager.class); } + @Bean + public TemplateDeployAsIsDetailsDao templateDeployAsIsDetailsDao() { + return Mockito.mock(TemplateDeployAsIsDetailsDao.class); + } + @Bean public SnapshotHelper snapshotHelper() { return Mockito.mock(SnapshotHelper.class); diff --git a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java index cfc8deba0a9..04119d41f0d 100644 --- a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java +++ b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java @@ -36,6 +36,7 @@ import static org.mockito.Mockito.never; import static org.mockito.Mockito.when; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -1079,7 +1080,7 @@ public class UserVmManagerImplTest { when(templateMock.isDeployAsIs()).thenReturn(false); when(templateMock.getFormat()).thenReturn(Storage.ImageFormat.QCOW2); when(templateMock.getUserDataId()).thenReturn(null); - Mockito.doNothing().when(vnfTemplateManager).validateVnfApplianceNics(any(), nullable(List.class)); + Mockito.doNothing().when(vnfTemplateManager).validateVnfApplianceNics(any(), nullable(List.class), nullable(Map.class)); ServiceOfferingJoinVO svcOfferingMock = Mockito.mock(ServiceOfferingJoinVO.class); when(serviceOfferingJoinDao.findById(anyLong())).thenReturn(svcOfferingMock); @@ -1091,7 +1092,7 @@ public class UserVmManagerImplTest { UserVm result = userVmManagerImpl.createVirtualMachine(deployVMCmd); assertEquals(userVmVoMock, result); - Mockito.verify(vnfTemplateManager).validateVnfApplianceNics(templateMock, null); + Mockito.verify(vnfTemplateManager).validateVnfApplianceNics(templateMock, null, Collections.emptyMap()); Mockito.verify(userVmManagerImpl).createBasicSecurityGroupVirtualMachine(any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), nullable(Boolean.class), any(), any(), any(), any(), any(), any(), any(), eq(true), any()); @@ -1335,7 +1336,7 @@ public class UserVmManagerImplTest { when(templateMock.isDeployAsIs()).thenReturn(false); when(templateMock.getFormat()).thenReturn(Storage.ImageFormat.QCOW2); when(templateMock.getUserDataId()).thenReturn(null); - Mockito.doNothing().when(vnfTemplateManager).validateVnfApplianceNics(any(), nullable(List.class)); + Mockito.doNothing().when(vnfTemplateManager).validateVnfApplianceNics(any(), nullable(List.class), nullable(Map.class)); ServiceOfferingJoinVO svcOfferingMock = Mockito.mock(ServiceOfferingJoinVO.class); when(serviceOfferingJoinDao.findById(anyLong())).thenReturn(svcOfferingMock); diff --git a/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java b/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java index 7f4344f30e4..673d0b7a48c 100644 --- a/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java +++ b/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java @@ -230,6 +230,13 @@ public class MockNetworkManagerImpl extends ManagerBase implements NetworkOrches return null; } + @Override + public Network createGuestNetwork(long networkOfferingId, String name, String displayText, Account owner, + PhysicalNetwork physicalNetwork, long zoneId, ACLType aclType, Pair vrIfaceMTUs) + throws InsufficientCapacityException, ConcurrentOperationException, ResourceAllocationException { + return null; + } + /* (non-Javadoc) * @see com.cloud.network.NetworkService#searchForNetworks(com.cloud.api.commands.ListNetworksCmd) */ diff --git a/server/src/test/java/org/apache/cloudstack/storage/template/VnfTemplateManagerImplTest.java b/server/src/test/java/org/apache/cloudstack/storage/template/VnfTemplateManagerImplTest.java index c3fa0d62604..b9565ebb292 100644 --- a/server/src/test/java/org/apache/cloudstack/storage/template/VnfTemplateManagerImplTest.java +++ b/server/src/test/java/org/apache/cloudstack/storage/template/VnfTemplateManagerImplTest.java @@ -228,25 +228,25 @@ public class VnfTemplateManagerImplTest { @Test public void testValidateVnfApplianceNicsWithRequiredNics() { List networkIds = Arrays.asList(200L, 201L); - vnfTemplateManagerImpl.validateVnfApplianceNics(template, networkIds); + vnfTemplateManagerImpl.validateVnfApplianceNics(template, networkIds, null); } @Test public void testValidateVnfApplianceNicsWithAllNics() { List networkIds = Arrays.asList(200L, 201L, 202L); - vnfTemplateManagerImpl.validateVnfApplianceNics(template, networkIds); + vnfTemplateManagerImpl.validateVnfApplianceNics(template, networkIds, null); } @Test(expected = InvalidParameterValueException.class) public void testValidateVnfApplianceNicsWithEmptyList() { List networkIds = new ArrayList<>(); - vnfTemplateManagerImpl.validateVnfApplianceNics(template, networkIds); + vnfTemplateManagerImpl.validateVnfApplianceNics(template, networkIds, null); } @Test(expected = InvalidParameterValueException.class) public void testValidateVnfApplianceNicsWithMissingNetworkId() { List networkIds = Arrays.asList(200L); - vnfTemplateManagerImpl.validateVnfApplianceNics(template, networkIds); + vnfTemplateManagerImpl.validateVnfApplianceNics(template, networkIds, null); } @Test diff --git a/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java b/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java index 5698632249d..c9bcb911000 100644 --- a/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java +++ b/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java @@ -1224,8 +1224,10 @@ public class SecondaryStorageManagerImpl extends ManagerBase implements Secondar if (dc.getDns2() != null) { buf.append(" dns2=").append(dc.getDns2()); } - String nfsVersion = imageStoreDetailsUtil != null ? imageStoreDetailsUtil.getNfsVersion(secStores.get(0).getId()) : null; - buf.append(" nfsVersion=").append(nfsVersion); + String nfsVersion = imageStoreDetailsUtil.getNfsVersion(secStores.get(0).getId()); + if (StringUtils.isNotBlank(nfsVersion)) { + buf.append(" nfsVersion=").append(nfsVersion); + } buf.append(" keystore_password=").append(VirtualMachineGuru.getEncodedString(PasswordGenerator.generateRandomPassword(16))); String bootArgs = buf.toString(); if (logger.isDebugEnabled()) { diff --git a/tools/marvin/marvin/lib/base.py b/tools/marvin/marvin/lib/base.py index 3d232d887c8..39a0ccb6dd8 100755 --- a/tools/marvin/marvin/lib/base.py +++ b/tools/marvin/marvin/lib/base.py @@ -1658,11 +1658,12 @@ class Template: # If template is ready, # template.status = Download Complete # Downloading - x% Downloaded + # Processing - Initial status # Error - Any other string if template.status == 'Download Complete' and template.isready: return - elif 'Downloaded' in template.status: + elif 'Downloaded' in template.status or template.status == 'Processing': retries = retries - 1 continue diff --git a/ui/public/config.json b/ui/public/config.json index 64d10284186..e3d1d30b95f 100644 --- a/ui/public/config.json +++ b/ui/public/config.json @@ -98,5 +98,6 @@ "multipleServer": false, "allowSettingTheme": true, "docHelpMappings": {}, - "notifyLatestCSVersion": true + "notifyLatestCSVersion": true, + "showSearchFilters": true } diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index 791091e8e2a..99873820d53 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -47,6 +47,7 @@ "label.acl.rules": "ACL rules", "label.acl.reason.description": "Enter the reason behind an ACL rule.", "label.aclid": "ACL", +"label.aclname": "ACL name", "label.acl.rule.name": "ACL rule name", "label.acquire.new.ip": "Acquire new IP", "label.acquire.new.secondary.ip": "Acquire new secondary IP", @@ -590,6 +591,8 @@ "label.copy.consoleurl": "Copy console URL to clipboard", "label.copyid": "Copy ID", "label.copy.password": "Copy password", +"label.copy.templates.from.other.secondary.storages": "Copy Templates from other storages instead of fetching from URLs", +"label.copy.templates.from.other.secondary.storages.add.zone": "Copy Templates from other storages", "label.core": "Core", "label.core.zone.type": "Core Zone type", "label.counter": "Counter", @@ -1254,6 +1257,7 @@ "label.isoname": "Attached ISO", "label.isos": "ISOs", "label.isostate": "ISO state", +"label.isourl": "ISO URL", "label.ispersistent": "Persistent ", "label.ispublic": "Public", "label.isready": "Ready", @@ -2526,8 +2530,8 @@ "label.vnf.app.action.reinstall": "Reinstall VNF Appliance", "label.vnf.cidr.list": "CIDR from which access to the VNF appliance's Management interface should be allowed from", "label.vnf.cidr.list.tooltip": "the CIDR list to forward traffic from to the VNF management interface. Multiple entries must be separated by a single comma character (,). The default value is 0.0.0.0/0.", -"label.vnf.configure.management": "Configure Firewall and Port Forwarding rules for VNF's management interfaces", -"label.vnf.configure.management.tooltip": "True by default, security group or network rules (source nat and firewall rules) will be configured for VNF management interfaces. False otherwise. Learn what rules are configured at http://docs.cloudstack.apache.org/en/latest/adminguide/networking/vnf_templates_appliances.html#deploying-vnf-appliances", +"label.vnf.configure.management": "Configure network rules for VNF's management interfaces", +"label.vnf.configure.management.tooltip": "False by default, security group or network rules (source nat and firewall rules) will be configured for VNF management interfaces. True otherwise. Learn what rules are configured at http://docs.cloudstack.apache.org/en/latest/adminguide/networking/vnf_templates_appliances.html#deploying-vnf-appliances", "label.vnf.detail.add": "Add VNF detail", "label.vnf.detail.remove": "Remove VNF detail", "label.vnf.details": "VNF Details", @@ -3017,7 +3021,7 @@ "message.desc.importmigratefromvmwarewizard": "By selecting an existing or external VMware Datacenter and an instance to import, CloudStack migrates the selected instance from VMware to KVM on a conversion host using virt-v2v and imports it into a KVM Cluster", "message.desc.primary.storage": "Each Cluster must contain one or more primary storage servers. We will add the first one now. Primary storage contains the disk volumes for all the Instances running on hosts in the cluster. Use any standards-compliant protocol that is supported by the underlying hypervisor.", "message.desc.reset.ssh.key.pair": "Please specify a ssh key pair that you would like to add to this Instance.", -"message.desc.secondary.storage": "Each Zone must have at least one NFS or secondary storage server. We will add the first one now. Secondary storage stores Instance Templates, ISO images, and Instance disk volume Snapshots. This server must be available to all hosts in the zone.

Provide the IP address and exported path.", +"message.desc.secondary.storage": "Each Zone must have at least one NFS or secondary storage server. We will add the first one now. Secondary storage stores Instance Templates, ISO images, and Instance disk volume Snapshots. This server must be available to all hosts in the zone.

Provide the IP address and exported path.

\"Copy templates from other secondary storages\" switch can be used to automatically copy existing templates from secondary storages in other zones instead of fetching from their URLs.", "message.desc.register.user.data": "Please fill in the following to register new User Data.", "message.desc.registered.user.data": "Registered a User Data.", "message.desc.zone": "A Zone is the largest organizational unit in CloudStack, and it typically corresponds to a single datacenter. Zones provide physical isolation and redundancy. A zone consists of one or more Pods (each of which contains hosts and primary storage servers) and a secondary storage server which is shared by all pods in the zone.", diff --git a/ui/public/locales/pt_BR.json b/ui/public/locales/pt_BR.json index 4d95b341ab4..1b51bc438e5 100644 --- a/ui/public/locales/pt_BR.json +++ b/ui/public/locales/pt_BR.json @@ -874,6 +874,7 @@ "label.isoname": "Imagem ISO plugada", "label.isos": "ISOs", "label.isostate": "Estado da ISO", +"label.isourl": "URL da ISO", "label.ispersistent": "Persistente", "label.ispublic": "P\u00fablico", "label.isready": "Pronto", diff --git a/ui/src/components/view/DedicateDomain.vue b/ui/src/components/view/DedicateDomain.vue index 0b3645ce418..4b8cc31ae46 100644 --- a/ui/src/components/view/DedicateDomain.vue +++ b/ui/src/components/view/DedicateDomain.vue @@ -18,52 +18,44 @@ diff --git a/ui/src/components/widgets/InfiniteScrollSelect.vue b/ui/src/components/widgets/InfiniteScrollSelect.vue index f97faf390f8..da780b66b80 100644 --- a/ui/src/components/widgets/InfiniteScrollSelect.vue +++ b/ui/src/components/widgets/InfiniteScrollSelect.vue @@ -41,8 +41,10 @@ - optionValueKey (String, optional): Property to use as the value for options (e.g., 'name'). Default is 'id' - optionLabelKey (String, optional): Property to use as the label for options (e.g., 'name'). Default is 'name' - defaultOption (Object, optional): Preselected object to include initially + - allowClear (Boolean, optional): Whether to allow clearing the selection. Default is false - showIcon (Boolean, optional): Whether to show icon for the options. Default is true - defaultIcon (String, optional): Icon to be shown when there is no resource icon for the option. Default is 'cloud-outlined' + - selectFirstOption (Boolean, optional): Whether to automatically select the first option when options are loaded. Default is false Events: - @change-option-value (Function): Emits the selected option value(s) when value(s) changes. Do not use @change as it will give warnings and may not work @@ -58,6 +60,7 @@ :filter-option="false" :loading="loading" show-search + :allowClear="allowClear" placeholder="Select" @search="onSearchTimed" @popupScroll="onScroll" @@ -75,9 +78,9 @@ - + - + @@ -124,6 +127,10 @@ export default { type: Object, default: null }, + allowClear: { + type: Boolean, + default: false + }, showIcon: { type: Boolean, default: true @@ -135,6 +142,10 @@ export default { pageSize: { type: Number, default: null + }, + selectFirstOption: { + type: Boolean, + default: false } }, data () { @@ -147,7 +158,8 @@ export default { searchTimer: null, scrollHandlerAttached: false, preselectedOptionValue: null, - successiveFetches: 0 + successiveFetches: 0, + hasAutoSelectedFirst: false } }, created () { @@ -166,6 +178,36 @@ export default { }, formattedSearchFooterMessage () { return `${this.$t('label.showing.results.for').replace('%x', this.searchQuery)}` + }, + selectableOptions () { + const currentValue = this.$attrs.value + // Only filter out null/empty options when the current value is also null/undefined/empty + // This prevents such options from being selected and allows the placeholder to show instead + if (currentValue === null || currentValue === undefined || currentValue === '') { + return this.options.filter(option => { + const optionValue = option[this.optionValueKey] + return optionValue !== null && optionValue !== undefined && optionValue !== '' + }) + } + // When a valid value is selected, show all options + return this.options + }, + apiOptionsCount () { + if (this.defaultOption) { + const defaultOptionValue = this.defaultOption[this.optionValueKey] + return this.options.filter(option => option[this.optionValueKey] !== defaultOptionValue).length + } + return this.options.length + }, + preselectedMatchValue () { + // Extract the first value from preselectedOptionValue if it's an array, otherwise return the value itself + if (!this.preselectedOptionValue) return null + return Array.isArray(this.preselectedOptionValue) ? this.preselectedOptionValue[0] : this.preselectedOptionValue + }, + preselectedMatch () { + // Find the matching option for the preselected value + if (!this.preselectedMatchValue) return null + return this.options.find(entry => entry[this.optionValueKey] === this.preselectedMatchValue) || null } }, watch: { @@ -210,6 +252,7 @@ export default { }).finally(() => { if (this.successiveFetches === 0) { this.loading = false + this.autoSelectFirstOptionIfNeeded() } }) }, @@ -220,11 +263,10 @@ export default { this.resetPreselectedOptionValue() return } - const matchValue = Array.isArray(this.preselectedOptionValue) ? this.preselectedOptionValue[0] : this.preselectedOptionValue - const match = this.options.find(entry => entry[this.optionValueKey] === matchValue) - if (!match) { + if (!this.preselectedMatch) { this.successiveFetches++ - if (this.options.length < this.totalCount) { + // Exclude defaultOption from count when comparing with totalCount + if (this.apiOptionsCount < this.totalCount) { this.fetchItems() } else { this.resetPreselectedOptionValue() @@ -232,7 +274,7 @@ export default { return } if (Array.isArray(this.preselectedOptionValue) && this.preselectedOptionValue.length > 1) { - this.preselectedOptionValue = this.preselectedOptionValue.filter(o => o !== match) + this.preselectedOptionValue = this.preselectedOptionValue.filter(o => o !== this.preselectedMatchValue) } else { this.resetPreselectedOptionValue() } @@ -246,6 +288,36 @@ export default { this.preselectedOptionValue = null this.successiveFetches = 0 }, + autoSelectFirstOptionIfNeeded () { + if (!this.selectFirstOption || this.hasAutoSelectedFirst) { + return + } + // Don't auto-select if there's a preselected value being fetched + if (this.preselectedOptionValue) { + return + } + const currentValue = this.$attrs.value + if (currentValue !== undefined && currentValue !== null && currentValue !== '') { + return + } + if (this.options.length === 0) { + return + } + if (this.searchQuery && this.searchQuery.length > 0) { + return + } + // Only auto-select after initial load is complete (no more successive fetches) + if (this.successiveFetches > 0) { + return + } + const firstOption = this.options[0] + if (firstOption) { + const firstValue = firstOption[this.optionValueKey] + this.hasAutoSelectedFirst = true + this.$emit('change-option-value', firstValue) + this.$emit('change-option', firstOption) + } + }, onSearchTimed (value) { clearTimeout(this.searchTimer) this.searchTimer = setTimeout(() => { @@ -264,7 +336,8 @@ export default { }, onScroll (e) { const nearBottom = e.target.scrollTop + e.target.clientHeight >= e.target.scrollHeight - 10 - const hasMore = this.options.length < this.totalCount + // Exclude defaultOption from count when comparing with totalCount + const hasMore = this.apiOptionsCount < this.totalCount if (nearBottom && hasMore && !this.loading) { this.fetchItems() } diff --git a/ui/src/config/section/image.js b/ui/src/config/section/image.js index 46dec2e1b24..3f8286c5fb1 100644 --- a/ui/src/config/section/image.js +++ b/ui/src/config/section/image.js @@ -60,9 +60,9 @@ export default { details: () => { var fields = ['name', 'id', 'displaytext', 'checksum', 'hypervisor', 'arch', 'format', 'ostypename', 'size', 'physicalsize', 'isready', 'passwordenabled', 'crossZones', 'templatetype', 'directdownload', 'deployasis', 'ispublic', 'isfeatured', 'isextractable', 'isdynamicallyscalable', 'crosszones', 'type', - 'account', 'domain', 'created', 'userdatadetails', 'userdatapolicy'] + 'account', 'domain', 'created', 'userdatadetails', 'userdatapolicy', 'url'] if (['Admin'].includes(store.getters.userInfo.roletype)) { - fields.push('templatetag', 'templatetype', 'url') + fields.push('templatetag', 'templatetype') } return fields }, @@ -372,7 +372,7 @@ export default { permission: ['listKubernetesSupportedVersions'], searchFilters: ['zoneid', 'minimumsemanticversion', 'arch'], columns: ['name', 'state', 'semanticversion', 'isostate', 'mincpunumber', 'minmemory', 'arch', 'zonename'], - details: ['name', 'semanticversion', 'supportsautoscaling', 'zoneid', 'zonename', 'isoid', 'isoname', 'isostate', 'arch', 'mincpunumber', 'minmemory', 'supportsha', 'state', 'created'], + details: ['name', 'semanticversion', 'supportsautoscaling', 'zoneid', 'zonename', 'isoid', 'isoname', 'isostate', 'arch', 'mincpunumber', 'minmemory', 'supportsha', 'state', 'created', 'isourl'], tabs: [ { name: 'details', diff --git a/ui/src/config/section/network.js b/ui/src/config/section/network.js index 30aae3a8deb..fbc044ff500 100644 --- a/ui/src/config/section/network.js +++ b/ui/src/config/section/network.js @@ -356,7 +356,10 @@ export default { permission: ['listVnfAppliances'], resourceType: 'UserVm', params: () => { - return { details: 'servoff,tmpl,nics', isvnf: true } + return { + details: 'group,nics,secgrp,tmpl,servoff,diskoff,iso,volume,affgrp,backoff,vnfnics', + isvnf: true + } }, columns: () => { const fields = ['name', 'state', 'ipaddress'] diff --git a/ui/src/utils/plugins.js b/ui/src/utils/plugins.js index a07f8178604..0ec957c8729 100644 --- a/ui/src/utils/plugins.js +++ b/ui/src/utils/plugins.js @@ -218,18 +218,19 @@ export const notifierPlugin = { if (error.response.status) { msg = `${i18n.global.t('message.request.failed')} (${error.response.status})` } - if (error.message) { - desc = error.message - } - if (error.response.headers && 'x-description' in error.response.headers) { + if (error.response.headers?.['x-description']) { desc = error.response.headers['x-description'] - } - if (desc === '' && error.response.data) { + } else if (error.response.data) { const responseKey = _.findKey(error.response.data, 'errortext') if (responseKey) { desc = error.response.data[responseKey].errortext + } else if (typeof error.response.data === 'string') { + desc = error.response.data } } + if (!desc && error.message) { + desc = error.message + } } let countNotify = store.getters.countNotify countNotify++ diff --git a/ui/src/views/AutogenView.vue b/ui/src/views/AutogenView.vue index a01e300c1c9..6acc81d6a02 100644 --- a/ui/src/views/AutogenView.vue +++ b/ui/src/views/AutogenView.vue @@ -106,6 +106,16 @@ @change-filter="changeFilter"/> + + + @@ -469,6 +479,7 @@ import ListView from '@/components/view/ListView' import ResourceView from '@/components/view/ResourceView' import ActionButton from '@/components/view/ActionButton' import SearchView from '@/components/view/SearchView' +import SearchFilter from '@/components/view/SearchFilter' import OsLogo from '@/components/widgets/OsLogo' import ResourceIcon from '@/components/view/ResourceIcon' import BulkActionProgress from '@/components/view/BulkActionProgress' @@ -482,6 +493,7 @@ export default { ListView, ActionButton, SearchView, + SearchFilter, BulkActionProgress, TooltipLabel, OsLogo, @@ -1126,6 +1138,42 @@ export default { eventBus.emit('action-closing', { action: this.currentAction }) this.closeAction() }, + getActiveFilters () { + const queryParams = Object.assign({}, this.$route.query) + const activeFilters = [] + for (const filter in queryParams) { + if (!filter.startsWith('tags[')) { + activeFilters.push({ + key: filter, + value: queryParams[filter], + isTag: false + }) + } else if (filter.endsWith('].key')) { + const tagIdx = filter.split('[')[1].split(']')[0] + const tagKey = queryParams[`tags[${tagIdx}].key`] + const tagValue = queryParams[`tags[${tagIdx}].value`] + activeFilters.push({ + key: tagKey, + value: tagValue, + isTag: true, + tagIdx: tagIdx + }) + } + } + return activeFilters + }, + removeFilter (filter) { + const queryParams = Object.assign({}, this.$route.query) + if (filter.isTag) { + delete queryParams[`tags[${filter.tagIdx}].key`] + delete queryParams[`tags[${filter.tagIdx}].value`] + } else { + delete queryParams[filter.key] + } + queryParams.page = '1' + queryParams.pagesize = String(this.pageSize) + this.$router.push({ query: queryParams }) + }, onRowSelectionChange (selection) { this.selectedRowKeys = selection if (selection?.length > 0) { diff --git a/ui/src/views/auth/ForgotPassword.vue b/ui/src/views/auth/ForgotPassword.vue index 87f2d1d0c33..1e817e01a6e 100644 --- a/ui/src/views/auth/ForgotPassword.vue +++ b/ui/src/views/auth/ForgotPassword.vue @@ -162,7 +162,7 @@ export default { api('forgotPassword', {}, 'POST', loginParams) .finally(() => { this.$message.success(this.$t('message.forgot.password.success')) - this.$router.push({ path: '/login' }).catch(() => {}) + this.$router.replace({ path: '/user/login' }) }) }).catch(error => { this.formRef.value.scrollToField(error.errorFields[0].name) diff --git a/ui/src/views/compute/DeployVM.vue b/ui/src/views/compute/DeployVM.vue index 97ef7bb5a96..72cb74399d1 100644 --- a/ui/src/views/compute/DeployVM.vue +++ b/ui/src/views/compute/DeployVM.vue @@ -2339,7 +2339,9 @@ export default { this.owner.domainid = null this.owner.projectid = OwnerOptions.selectedProject } - this.resetData() + if (OwnerOptions.initialized) { + this.resetData() + } }, fetchZones (zoneId, listZoneAllow) { this.zones = [] diff --git a/ui/src/views/compute/DeployVnfAppliance.vue b/ui/src/views/compute/DeployVnfAppliance.vue index 1117413d710..9b09de5a186 100644 --- a/ui/src/views/compute/DeployVnfAppliance.vue +++ b/ui/src/views/compute/DeployVnfAppliance.vue @@ -372,6 +372,7 @@
@@ -1293,7 +1294,8 @@ export default { return tabList }, showVnfNicsSection () { - return this.networks && this.networks.length > 0 && this.vm.templateid && this.templateVnfNics && this.templateVnfNics.length > 0 + return ((this.networks && this.networks.length > 0) || (this.templateNics && this.templateNics.length > 0)) && + this.vm.templateid && this.templateVnfNics && this.templateVnfNics.length > 0 }, showVnfConfigureManagement () { const managementDeviceIds = [] @@ -1303,9 +1305,14 @@ export default { } } for (const deviceId of managementDeviceIds) { + if (this.templateNics && this.templateNics[deviceId] && + ((this.templateNics[deviceId].selectednetworktype === 'Isolated' && this.templateNics[deviceId].selectednetworkvpcid === undefined) || + (this.templateNics[deviceId].selectednetworktype === 'Shared' && this.templateNics[deviceId].selectednetworkwithsg))) { + return true + } if (this.vnfNicNetworks && this.vnfNicNetworks[deviceId] && ((this.vnfNicNetworks[deviceId].type === 'Isolated' && this.vnfNicNetworks[deviceId].vpcid === undefined) || - (this.vnfNicNetworks[deviceId].type === 'Shared' && this.zone.securitygroupsenabled))) { + (this.vnfNicNetworks[deviceId].type === 'Shared' && this.vnfNicNetworks[deviceId].service.filter(svc => svc.name === 'SecurityGroupProvider')))) { return true } } @@ -2005,7 +2012,7 @@ export default { // All checked networks should be used and only once. // Required NIC must be associated to a network // DeviceID must be consequent - if (this.templateVnfNics && this.templateVnfNics.length > 0) { + if (this.templateVnfNics && this.templateVnfNics.length > 0 && (!this.templateNics || this.templateNics.length === 0)) { let nextDeviceId = 0 const usedNetworkIds = [] const keys = Object.keys(this.vnfNicNetworks) @@ -2629,6 +2636,9 @@ export default { var network = this.options.networks[Math.min(i, this.options.networks.length - 1)] nic.selectednetworkid = network.id nic.selectednetworkname = network.name + nic.selectednetworktype = network.type + nic.selectednetworkvpcid = network.vpcid + nic.selectednetworkwithsg = network.service.filter(svc => svc.name === 'SecurityGroupProvider').length > 0 this.nicToNetworkSelection.push({ nic: nic.id, network: network.id }) } } diff --git a/ui/src/views/compute/wizard/OwnershipSelection.vue b/ui/src/views/compute/wizard/OwnershipSelection.vue index a2c7ea4c1f8..b29069628db 100644 --- a/ui/src/views/compute/wizard/OwnershipSelection.vue +++ b/ui/src/views/compute/wizard/OwnershipSelection.vue @@ -19,7 +19,7 @@ domain.id) const ownerDomainId = this.$store.getters.project?.domainid || this.$store.getters.userInfo.domainid this.selectedDomain = domainIds?.includes(ownerDomainId) ? ownerDomainId : this.domains?.[0]?.id - this.changeDomain() + this.fetchOwnerData() }) .catch((error) => { this.$notifyError(error) @@ -186,8 +188,13 @@ export default { this.loading = false }) }, + incrementAndGetRequestToken () { + this.requestToken += 1 + return this.requestToken + }, fetchAccounts () { this.loading = true + const currentToken = this.incrementAndGetRequestToken() api('listAccounts', { response: 'json', domainId: this.selectedDomain, @@ -196,6 +203,9 @@ export default { isrecursive: false }) .then((response) => { + if (currentToken !== this.requestToken) { + return + } this.accounts = response.listaccountsresponse.account || [] if (this.override?.accounts && this.accounts) { this.accounts = this.accounts.filter(item => this.override.accounts.has(item.name)) @@ -214,10 +224,12 @@ export default { }) .finally(() => { this.loading = false + this.initialized = true }) }, fetchProjects () { this.loading = true + const currentToken = this.incrementAndGetRequestToken() api('listProjects', { response: 'json', domainId: this.selectedDomain, @@ -227,6 +239,9 @@ export default { isrecursive: false }) .then((response) => { + if (currentToken !== this.requestToken) { + return + } this.projects = response.listprojectsresponse.project if (this.override?.projects && this.projects) { this.projects = this.projects.filter(item => this.override.projects.has(item.id)) @@ -240,9 +255,14 @@ export default { }) .finally(() => { this.loading = false + this.initialized = true }) }, - changeDomain () { + changeAccountTypeOrDomain () { + this.initialized = true + this.fetchOwnerData() + }, + fetchOwnerData () { if (this.selectedAccountType === 'Account') { this.fetchAccounts() } else { diff --git a/ui/src/views/compute/wizard/VnfNicsSelection.vue b/ui/src/views/compute/wizard/VnfNicsSelection.vue index fdd5276b4f6..40bdc1c676a 100644 --- a/ui/src/views/compute/wizard/VnfNicsSelection.vue +++ b/ui/src/views/compute/wizard/VnfNicsSelection.vue @@ -50,6 +50,7 @@ - - - - - - {{ domain.path || domain.name || domain.description }} - - - + @change-option-value="handleDomainChange" /> - - - - - - {{ item.name }} - - - + api="listAccounts" + :apiParams="accountsApiParams" + resourceType="account" + optionValueKey="name" + optionLabelKey="name" + defaultIcon="team-outlined" + :placeholder="apiParams.account.description" />