etan39
/
cloudstack
mirror of https://github.com/apache/cloudstack.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Deal with Storage Manager tech debt

This commit is contained in:
Daan Hoogland 2025-12-22 19:56:33 +01:00 committed by dahn
parent 5bf869c803
commit 7d7e5dfca8
15 changed files with 375 additions and 644 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
api/src/main/java/com/cloud/storage/StorageService.java
View File

@ -17,7 +17,6 @@
package com.cloud.storage;
import java.net.UnknownHostException;
import java.util.Map;
import org.apache.cloudstack.api.command.admin.storage.CancelPrimaryStorageMaintenanceCmd;
@ -35,10 +34,8 @@ import org.apache.cloudstack.api.command.admin.storage.UpdateImageStoreCmd;
import org.apache.cloudstack.api.command.admin.storage.UpdateStoragePoolCmd;
import com.cloud.exception.DiscoveryException;
import com.cloud.exception.InsufficientCapacityException;
import com.cloud.exception.InvalidParameterValueException;
import com.cloud.exception.PermissionDeniedException;
import com.cloud.exception.ResourceInUseException;
import com.cloud.exception.ResourceUnavailableException;
import org.apache.cloudstack.api.command.admin.storage.heuristics.CreateSecondaryStorageSelectorCmd;
import org.apache.cloudstack.api.command.admin.storage.heuristics.RemoveSecondaryStorageSelectorCmd;
@ -55,12 +52,9 @@ public interface StorageService {
* storage pool.
* @return
* The StoragePool created.
* @throws ResourceInUseException
* @throws IllegalArgumentException
* @throws UnknownHostException
* @throws ResourceUnavailableException
*/
StoragePool createPool(CreateStoragePoolCmd cmd) throws ResourceInUseException, IllegalArgumentException, UnknownHostException, ResourceUnavailableException;
StoragePool createPool(CreateStoragePoolCmd cmd) throws IllegalArgumentException;
ImageStore createSecondaryStagingStore(CreateSecondaryStagingStoreCmd cmd);
@ -79,10 +73,8 @@ public interface StorageService {
* @param primaryStorageId
* - the primaryStorageId
* @return the primary storage pool
* @throws ResourceUnavailableException
* @throws InsufficientCapacityException
*/
StoragePool preparePrimaryStorageForMaintenance(Long primaryStorageId) throws ResourceUnavailableException, InsufficientCapacityException;
StoragePool preparePrimaryStorageForMaintenance(Long primaryStorageId);
/**
* Complete maintenance for primary storage
@ -108,7 +100,7 @@ public interface StorageService {
boolean deleteSecondaryStagingStore(DeleteSecondaryStagingStoreCmd cmd);
ImageStore discoverImageStore(String name, String url, String providerName, Long zoneId, Map details) throws IllegalArgumentException, DiscoveryException, InvalidParameterValueException;
ImageStore discoverImageStore(String name, String url, String providerName, Long zoneId, Map<String, String> details) throws IllegalArgumentException, InvalidParameterValueException;
/**
* Migrate existing NFS to use object store.
@ -134,7 +126,7 @@ public interface StorageService {
void removeSecondaryStorageHeuristic(RemoveSecondaryStorageSelectorCmd cmd);
ObjectStore discoverObjectStore(String name, String url, Long size, String providerName, Map details) throws IllegalArgumentException, DiscoveryException, InvalidParameterValueException;
ObjectStore discoverObjectStore(String name, String url, Long size, String providerName, Map<String, String> details) throws IllegalArgumentException, DiscoveryException, InvalidParameterValueException;
boolean deleteObjectStore(DeleteObjectStoragePoolCmd cmd);

24
api/src/main/java/org/apache/cloudstack/api/command/admin/host/AddSecondaryStorageCmd.java
View File

@ -26,7 +26,6 @@ import org.apache.cloudstack.api.ServerApiException;
import org.apache.cloudstack.api.response.ImageStoreResponse;
import org.apache.cloudstack.api.response.ZoneResponse;
import com.cloud.exception.DiscoveryException;
import com.cloud.storage.ImageStore;
import com.cloud.user.Account;
@ -67,20 +66,15 @@ public class AddSecondaryStorageCmd extends BaseCmd {
@Override
public void execute(){
try{
ImageStore result = _storageService.discoverImageStore(null, getUrl(), "NFS", getZoneId(), null);
ImageStoreResponse storeResponse = null;
if (result != null ) {
storeResponse = _responseGenerator.createImageStoreResponse(result);
storeResponse.setResponseName(getCommandName());
storeResponse.setObjectName("secondarystorage");
setResponseObject(storeResponse);
} else {
throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to add secondary storage");
}
} catch (DiscoveryException ex) {
logger.warn("Exception: ", ex);
throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage());
ImageStore result = _storageService.discoverImageStore(null, getUrl(), "NFS", getZoneId(), null);
ImageStoreResponse storeResponse = null;
if (result != null ) {
storeResponse = _responseGenerator.createImageStoreResponse(result);
storeResponse.setResponseName(getCommandName());
storeResponse.setObjectName("secondarystorage");
setResponseObject(storeResponse);
} else {
throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to add secondary storage");
}
}
}

32
api/src/main/java/org/apache/cloudstack/api/command/admin/storage/AddImageStoreCmd.java
View File

@ -18,7 +18,6 @@ package org.apache.cloudstack.api.command.admin.storage;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
@ -31,7 +30,6 @@ import org.apache.cloudstack.api.ServerApiException;
import org.apache.cloudstack.api.response.ImageStoreResponse;
import org.apache.cloudstack.api.response.ZoneResponse;
import com.cloud.exception.DiscoveryException;
import com.cloud.storage.ImageStore;
import com.cloud.user.Account;
@ -79,11 +77,10 @@ public class AddImageStoreCmd extends BaseCmd {
public Map<String, String> getDetails() {
Map<String, String> detailsMap = null;
if (details != null && !details.isEmpty()) {
detailsMap = new HashMap<String, String>();
detailsMap = new HashMap<>();
Collection<?> props = details.values();
Iterator<?> iter = props.iterator();
while (iter.hasNext()) {
HashMap<String, String> detail = (HashMap<String, String>)iter.next();
for (Object prop : props) {
HashMap<String, String> detail = (HashMap<String, String>) prop;
String key = detail.get("key");
String value = detail.get("value");
detailsMap.put(key, value);
@ -123,20 +120,15 @@ public class AddImageStoreCmd extends BaseCmd {
@Override
public void execute(){
try{
ImageStore result = _storageService.discoverImageStore(getName(), getUrl(), getProviderName(), getZoneId(), getDetails());
ImageStoreResponse storeResponse = null;
if (result != null) {
storeResponse = _responseGenerator.createImageStoreResponse(result);
storeResponse.setResponseName(getCommandName());
storeResponse.setObjectName("imagestore");
setResponseObject(storeResponse);
} else {
throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to add secondary storage");
}
} catch (DiscoveryException ex) {
logger.warn("Exception: ", ex);
throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage());
ImageStore result = _storageService.discoverImageStore(getName(), getUrl(), getProviderName(), getZoneId(), getDetails());
ImageStoreResponse storeResponse;
if (result != null) {
storeResponse = _responseGenerator.createImageStoreResponse(result);
storeResponse.setResponseName(getCommandName());
storeResponse.setObjectName("imagestore");
setResponseObject(storeResponse);
} else {
throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to add secondary storage");
}
}
}

44
api/src/main/java/org/apache/cloudstack/api/command/admin/storage/AddImageStoreS3CMD.java
View File

@ -48,15 +48,14 @@ import org.apache.cloudstack.api.ServerApiException;
import org.apache.cloudstack.api.response.ImageStoreResponse;
import com.cloud.exception.ConcurrentOperationException;
import com.cloud.exception.DiscoveryException;
import com.cloud.exception.InsufficientCapacityException;
import com.cloud.exception.NetworkRuleConflictException;
import com.cloud.exception.ResourceAllocationException;
import com.cloud.exception.ResourceUnavailableException;
import com.cloud.storage.ImageStore;
@APICommand(name = "addImageStoreS3", description = "Adds S3 Image Store", responseObject = ImageStoreResponse.class, since = "4.7.0",
requestHasSensitiveInfo = true, responseHasSensitiveInfo = false)
@APICommand(name = "addImageStoreS3", description = "Adds S3 Image Store", responseObject = ImageStoreResponse.class,
since = "4.7.0", responseHasSensitiveInfo = false)
public final class AddImageStoreS3CMD extends BaseCmd implements ClientOptions {
private static final String s_name = "addImageStoreS3Response";
@ -73,32 +72,32 @@ public final class AddImageStoreS3CMD extends BaseCmd implements ClientOptions {
@Parameter(name = S3_BUCKET_NAME, type = STRING, required = true, description = "Name of the storage bucket")
private String bucketName;
@Parameter(name = S3_SIGNER, type = STRING, required = false, description = "Signer Algorithm to use, either S3SignerType or AWSS3V4SignerType")
@Parameter(name = S3_SIGNER, type = STRING, description = "Signer Algorithm to use, either S3SignerType or AWSS3V4SignerType")
private String signer;
@Parameter(name = S3_HTTPS_FLAG, type = BOOLEAN, required = false, description = "Use HTTPS instead of HTTP")
@Parameter(name = S3_HTTPS_FLAG, type = BOOLEAN, description = "Use HTTPS instead of HTTP")
private Boolean httpsFlag;
@Parameter(name = S3_CONNECTION_TIMEOUT, type = INTEGER, required = false, description = "Connection timeout (milliseconds)")
@Parameter(name = S3_CONNECTION_TIMEOUT, type = INTEGER, description = "Connection timeout (milliseconds)")
private Integer connectionTimeout;
@Parameter(name = S3_MAX_ERROR_RETRY, type = INTEGER, required = false, description = "Maximum number of times to retry on error")
@Parameter(name = S3_MAX_ERROR_RETRY, type = INTEGER, description = "Maximum number of times to retry on error")
private Integer maxErrorRetry;
@Parameter(name = S3_SOCKET_TIMEOUT, type = INTEGER, required = false, description = "Socket timeout (milliseconds)")
@Parameter(name = S3_SOCKET_TIMEOUT, type = INTEGER, description = "Socket timeout (milliseconds)")
private Integer socketTimeout;
@Parameter(name = S3_CONNECTION_TTL, type = INTEGER, required = false, description = "Connection TTL (milliseconds)")
@Parameter(name = S3_CONNECTION_TTL, type = INTEGER, description = "Connection TTL (milliseconds)")
private Integer connectionTtl;
@Parameter(name = S3_USE_TCP_KEEPALIVE, type = BOOLEAN, required = false, description = "Whether TCP keep-alive is used")
@Parameter(name = S3_USE_TCP_KEEPALIVE, type = BOOLEAN, description = "Whether TCP keep-alive is used")
private Boolean useTCPKeepAlive;
@Override
public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException,
ResourceAllocationException, NetworkRuleConflictException {
Map<String, String> dm = new HashMap();
Map<String, String> dm = new HashMap<>();
dm.put(ApiConstants.S3_ACCESS_KEY, getAccessKey());
dm.put(ApiConstants.S3_SECRET_KEY, getSecretKey());
@ -127,20 +126,15 @@ public final class AddImageStoreS3CMD extends BaseCmd implements ClientOptions {
dm.put(ApiConstants.S3_USE_TCP_KEEPALIVE, getUseTCPKeepAlive().toString());
}
try{
ImageStore result = _storageService.discoverImageStore(null, null, "S3", null, dm);
ImageStoreResponse storeResponse;
if (result != null) {
storeResponse = _responseGenerator.createImageStoreResponse(result);
storeResponse.setResponseName(getCommandName());
storeResponse.setObjectName("imagestore");
setResponseObject(storeResponse);
} else {
throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to add S3 Image Store.");
}
} catch (DiscoveryException ex) {
logger.warn("Exception: ", ex);
throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage());
ImageStore result = _storageService.discoverImageStore(null, null, "S3", null, dm);
ImageStoreResponse storeResponse;
if (result != null) {
storeResponse = _responseGenerator.createImageStoreResponse(result);
storeResponse.setResponseName(getCommandName());
storeResponse.setObjectName("imagestore");
setResponseObject(storeResponse);
} else {
throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to add S3 Image Store.");
}
}

74
api/src/main/java/org/apache/cloudstack/api/command/admin/storage/CreateStoragePoolCmd.java
View File

@ -16,7 +16,6 @@
// under the License.
package org.apache.cloudstack.api.command.admin.storage;
import java.net.UnknownHostException;
import java.util.Map;
@ -31,8 +30,6 @@ import org.apache.cloudstack.api.response.PodResponse;
import org.apache.cloudstack.api.response.StoragePoolResponse;
import org.apache.cloudstack.api.response.ZoneResponse;
import com.cloud.exception.ResourceInUseException;
import com.cloud.exception.ResourceUnavailableException;
import com.cloud.storage.StoragePool;
import com.cloud.user.Account;
@ -46,53 +43,85 @@ public class CreateStoragePoolCmd extends BaseCmd {
//////////////// API parameters /////////////////////
/////////////////////////////////////////////////////
@Parameter(name = ApiConstants.CLUSTER_ID, type = CommandType.UUID, entityType = ClusterResponse.class, description = "The cluster ID for the storage pool")
@Parameter(name = ApiConstants.CLUSTER_ID,
type = CommandType.UUID,
entityType = ClusterResponse.class,
description = "The cluster ID for the storage pool")
private Long clusterId;
@Parameter(name = ApiConstants.DETAILS, type = CommandType.MAP, description = "The details for the storage pool")
@Parameter(name = ApiConstants.DETAILS,
type = CommandType.MAP,
description = "The details for the storage pool")
private Map details;
@Parameter(name = ApiConstants.NAME, type = CommandType.STRING, required = true, description = "The name for the storage pool")
@Parameter(name = ApiConstants.NAME,
type = CommandType.STRING,
required = true,
description = "The name for the storage pool")
private String storagePoolName;
@Parameter(name = ApiConstants.POD_ID, type = CommandType.UUID, entityType = PodResponse.class, description = "The Pod ID for the storage pool")
@Parameter(name = ApiConstants.POD_ID,
type = CommandType.UUID,
entityType = PodResponse.class,
description = "The Pod ID for the storage pool")
private Long podId;
@Parameter(name = ApiConstants.TAGS, type = CommandType.STRING, description = "The tags for the storage pool")
@Parameter(name = ApiConstants.TAGS,
type = CommandType.STRING,
description = "The tags for the storage pool")
private String tags;
@Parameter(name = ApiConstants.STORAGE_ACCESS_GROUPS, type = CommandType.STRING,
@Parameter(name = ApiConstants.STORAGE_ACCESS_GROUPS,
type = CommandType.STRING,
description = "comma separated list of storage access groups for connecting to hosts having those specific groups", since = "4.21.0")
private String storageAccessGroups;
@Parameter(name = ApiConstants.URL, type = CommandType.STRING, required = true, description = "The URL of the storage pool")
@Parameter(name = ApiConstants.URL,
type = CommandType.STRING,
required = true,
description = "The URL of the storage pool")
private String url;
@Parameter(name = ApiConstants.ZONE_ID, type = CommandType.UUID, entityType = ZoneResponse.class, required = true, description = "The Zone ID for the storage pool")
@Parameter(name = ApiConstants.ZONE_ID,
type = CommandType.UUID,
entityType = ZoneResponse.class,
required = true,
description = "The Zone ID for the storage pool")
private Long zoneId;
@Parameter(name = ApiConstants.PROVIDER, type = CommandType.STRING, required = false, description = "The storage provider name")
@Parameter(name = ApiConstants.PROVIDER,
type = CommandType.STRING,
description = "The storage provider name")
private String storageProviderName;
@Parameter(name = ApiConstants.SCOPE, type = CommandType.STRING, required = false, description = "The scope of the storage: cluster or zone")
@Parameter(name = ApiConstants.SCOPE,
type = CommandType.STRING,
description = "The scope of the storage: cluster or zone")
private String scope;
@Parameter(name = ApiConstants.MANAGED, type = CommandType.BOOLEAN, required = false, description = "Whether the storage should be managed by CloudStack")
@Parameter(name = ApiConstants.MANAGED,
type = CommandType.BOOLEAN,
description = "Whether the storage should be managed by CloudStack")
private Boolean managed;
@Parameter(name = ApiConstants.CAPACITY_IOPS, type = CommandType.LONG, required = false, description = "IOPS CloudStack can provision from this storage pool")
@Parameter(name = ApiConstants.CAPACITY_IOPS,
type = CommandType.LONG,
description = "IOPS CloudStack can provision from this storage pool")
private Long capacityIops;
@Parameter(name = ApiConstants.CAPACITY_BYTES, type = CommandType.LONG, required = false, description = "Bytes CloudStack can provision from this storage pool")
@Parameter(name = ApiConstants.CAPACITY_BYTES,
type = CommandType.LONG,
description = "Bytes CloudStack can provision from this storage pool")
private Long capacityBytes;
@Parameter(name = ApiConstants.HYPERVISOR,
type = CommandType.STRING,
required = false,
description = "Hypervisor type of the hosts in zone that will be attached to this storage pool. KVM, VMware supported as of now.")
private String hypervisor;
@Parameter(name = ApiConstants.IS_TAG_A_RULE, type = CommandType.BOOLEAN, description = ApiConstants.PARAMETER_DESCRIPTION_IS_TAG_A_RULE)
@Parameter(name = ApiConstants.IS_TAG_A_RULE,
type = CommandType.BOOLEAN,
description = ApiConstants.PARAMETER_DESCRIPTION_IS_TAG_A_RULE)
private Boolean isTagARule;
/////////////////////////////////////////////////////
@ -175,15 +204,6 @@ public class CreateStoragePoolCmd extends BaseCmd {
} else {
throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to add storage pool");
}
} catch (ResourceUnavailableException ex1) {
logger.warn("Exception: ", ex1);
throw new ServerApiException(ApiErrorCode.RESOURCE_UNAVAILABLE_ERROR, ex1.getMessage());
} catch (ResourceInUseException ex2) {
logger.warn("Exception: ", ex2);
throw new ServerApiException(ApiErrorCode.RESOURCE_IN_USE_ERROR, ex2.getMessage());
} catch (UnknownHostException ex3) {
logger.warn("Exception: ", ex3);
throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex3.getMessage());
} catch (Exception ex4) {
throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex4.getMessage());
}

2
api/src/main/java/org/apache/cloudstack/api/command/admin/storage/UpdateStoragePoolCmd.java
View File

@ -131,7 +131,7 @@ public class UpdateStoragePoolCmd extends BaseCmd {
return ApiCommandResourceType.StoragePool;
}
public Map<String,String> getDetails() {
public Map<String,Object> getDetails() {
return details;
}

26
api/src/main/java/org/apache/cloudstack/api/command/admin/swift/AddSwiftCmd.java
View File

@ -28,7 +28,6 @@ import org.apache.cloudstack.api.Parameter;
import org.apache.cloudstack.api.ServerApiException;
import org.apache.cloudstack.api.response.ImageStoreResponse;
import com.cloud.exception.DiscoveryException;
import com.cloud.storage.ImageStore;
import com.cloud.user.Account;
@ -83,25 +82,20 @@ public class AddSwiftCmd extends BaseCmd {
@Override
public void execute() {
Map<String, String> dm = new HashMap<String, String>();
Map<String, String> dm = new HashMap<>();
dm.put(ApiConstants.ACCOUNT, getAccount());
dm.put(ApiConstants.USERNAME, getUsername());
dm.put(ApiConstants.KEY, getKey());
try{
ImageStore result = _storageService.discoverImageStore(null, getUrl(), "Swift", null, dm);
ImageStoreResponse storeResponse = null;
if (result != null) {
storeResponse = _responseGenerator.createImageStoreResponse(result);
storeResponse.setResponseName(getCommandName());
storeResponse.setObjectName("secondarystorage");
setResponseObject(storeResponse);
} else {
throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to add Swift secondary storage");
}
} catch (DiscoveryException ex) {
logger.warn("Exception: ", ex);
throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage());
ImageStore result = _storageService.discoverImageStore(null, getUrl(), "Swift", null, dm);
ImageStoreResponse storeResponse;
if (result != null) {
storeResponse = _responseGenerator.createImageStoreResponse(result);
storeResponse.setResponseName(getCommandName());
storeResponse.setObjectName("secondarystorage");
setResponseObject(storeResponse);
} else {
throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to add Swift secondary storage");
}
}
}

47
engine/components-api/src/main/java/com/cloud/configuration/ConfigurationManager.java
View File

@ -71,6 +71,53 @@ public interface ConfigurationManager {
"Weight for CPU (as a value between 0 and 1) applied to compute capacity for Pods, Clusters and Hosts for COMBINED capacityType for ordering. Weight for RAM will be (1 - weight of CPU)",
true, ConfigKey.Scope.Global);
ConfigKey<Integer> ExpungeDelay = new ConfigKey<>(
ConfigKey.CATEGORY_ADVANCED,
Integer.class,
"expunge.delay",
"86400",
"Determines how long (in seconds) to wait before actually expunging destroyed vm. The default value = the default value of expunge.interval",
false,
ConfigKey.Scope.Global,
null);
ConfigKey<Integer> ExpungeInterval = new ConfigKey<>(
ConfigKey.CATEGORY_ADVANCED,
Integer.class,
"expunge.interval",
"86400",
"The interval (in seconds) to wait before running the expunge thread.",
false,
ConfigKey.Scope.Global,
null);
ConfigKey<Integer> ExpungeWorkers = new ConfigKey<>(
ConfigKey.CATEGORY_ADVANCED,
Integer.class,
"expunge.workers",
"10",
"Number of workers performing expunge",
false,
ConfigKey.Scope.Global,
null);
ConfigKey<Integer> ExtractURLCleanUpInterval = new ConfigKey<>(
ConfigKey.CATEGORY_ADVANCED,
Integer.class,
"extract.url.cleanup.interval",
"7200",
"The interval (in seconds) to wait before cleaning up the extract URL's ",
false,
ConfigKey.Scope.Global,
null);
ConfigKey<Integer> ExtractURLExpirationInterval = new ConfigKey<>(
ConfigKey.CATEGORY_ADVANCED,
Integer.class,
"extract.url.expiration.interval",
"14400",
"The life of an extract URL after which it is deleted ",
false,
ConfigKey.Scope.Global,
null);
/**
* Is this for a VPC
* @param offering the offering to check

60
engine/components-api/src/main/java/com/cloud/storage/StorageManager.java
View File

@ -232,6 +232,43 @@ public interface StorageManager extends StorageService {
"Storage", "true", "Allow SSVMs to try copying public templates from one secondary storage to another instead of downloading them from the source.",
true, ConfigKey.Scope.Zone, null);
ConfigKey<Integer> VmDiskThrottlingIopsReadRate = new ConfigKey<>(
Integer.class,
"vm.disk.throttling.iops_read_rate",
"Storage",
"0",
"Default disk I/O read rate in requests per second allowed in User vm's disk.",
true,
ConfigKey.Scope.Global,
null);
ConfigKey<Integer> VmDiskThrottlingIopsWriteRate = new ConfigKey<>(
Integer.class,
"vm.disk.throttling.iops_write_rate",
"Storage",
"0",
"Default disk I/O writerate in requests per second allowed in User vm's disk.",
true,
ConfigKey.Scope.Global,
null);
ConfigKey<Integer> VmDiskThrottlingBytesReadRate = new ConfigKey<>(
Integer.class,
"vm.disk.throttling.bytes_read_rate",
"Storage",
"0",
"Default disk I/O read rate in bytes per second allowed in User vm's disk.",
true,
ConfigKey.Scope.Global,
null);
ConfigKey<Integer> VmDiskThrottlingBytesWriteRate = new ConfigKey<>(
Integer.class,
"vm.disk.throttling.bytes_write_rate",
"Advanced",
"0",
"Default disk I/O writerate in bytes per second allowed in User vm's disk.",
true,
ConfigKey.Scope.Global,
null);
/**
* should we execute in sequence not involving any storages?
* @return true if commands should execute in sequence
@ -241,8 +278,8 @@ public interface StorageManager extends StorageService {
}
static boolean shouldExecuteInSequenceOnVmware(Long srcStoreId, Long dstStoreId) {
final Boolean fullClone = getFullCloneConfiguration(srcStoreId) || getFullCloneConfiguration(dstStoreId);
final Boolean allowParallel = getAllowParallelExecutionConfiguration();
final boolean fullClone = getFullCloneConfiguration(srcStoreId) || getFullCloneConfiguration(dstStoreId);
final boolean allowParallel = getAllowParallelExecutionConfiguration();
return fullClone && !allowParallel;
}
@ -284,10 +321,6 @@ public interface StorageManager extends StorageService {
boolean canPoolProvideStorageStats(StoragePool pool);
boolean poolProvidesCustomStorageStats(StoragePool pool);
Map<String, String> getCustomStorageStats(StoragePool pool);
/**
* Checks if a host has running VMs that are using its local storage pool.
* @return true if local storage is active on the host
@ -300,8 +333,6 @@ public interface StorageManager extends StorageService {
*/
void cleanupStorage(boolean recurring);
String getPrimaryStorageNameLabel(VolumeVO volume);
void createCapacityEntry(StoragePoolVO storagePool, short capacityType, long allocated);
Answer sendToPool(StoragePool pool, long[] hostIdsToTryFirst, Command cmd) throws StorageUnavailableException;
@ -314,8 +345,6 @@ public interface StorageManager extends StorageService {
CapacityVO getStoragePoolUsedStats(Long zoneId, Long podId, Long clusterId, List<Long> poolIds);
List<StoragePoolVO> ListByDataCenterHypervisor(long datacenterId, HypervisorType type);
List<VMInstanceVO> listByStoragePool(long storagePoolId);
StoragePoolVO findLocalStorageOnHost(long hostId);
@ -328,11 +357,9 @@ public interface StorageManager extends StorageService {
boolean canHostPrepareStoragePoolAccess(Host host, StoragePool pool);
boolean canDisconnectHostFromStoragePool(Host host, StoragePool pool);
Host getHost(long hostId);
Host updateSecondaryStorage(long secStorageId, String newUrl);
void updateSecondaryStorage(long secStorageId, String newUrl);
void removeStoragePoolFromCluster(long hostId, String iScsiName, StoragePool storagePool);
@ -351,24 +378,19 @@ public interface StorageManager extends StorageService {
/**
* This comment is relevant to managed storage only.
*
* Long clusterId = only used for managed storage
*
* Some managed storage can be more efficient handling VM templates (via cloning) if it knows the capabilities of the compute cluster it is dealing with.
* If the compute cluster supports UUID resigning and the storage system can clone a volume from a volume, then this determines how much more space a
* new root volume (that makes use of a template) will take up on the storage system.
*
* For example, if a storage system can clone a volume from a volume and the compute cluster supports UUID resigning (relevant for hypervisors like
* XenServer and ESXi that put virtual disks in clustered file systems), then the storage system will need to determine if it already has a copy of
* the template or if it will need to create one first before cloning the template to a new volume to be used for the new root disk (assuming the root
* disk is being deployed from a template). If the template doesn't already exists on the storage system, then you need to take into consideration space
* disk is being deployed from a template). If the template doesn't already exist on the storage system, then you need to take into consideration space
* required for that template (stored in one volume) and space required for a new volume created from that template volume (for your new root volume).
*
* If UUID resigning is not available in the compute cluster or the storage system doesn't support cloning a volume from a volume, then for each new
* root disk that uses a template, CloudStack will have the template be copied down to a newly created volume on the storage system (i.e. no need
* to take into consideration the possible need to first create a volume on the storage system for a template that will be used for the root disk
* via cloning).
*
* Cloning volumes on the back-end instead of copying down a new template for each new volume helps to alleviate load on the hypervisors.
*/
boolean storagePoolHasEnoughSpace(List<Pair<Volume, DiskProfile>> volume, StoragePool pool, Long clusterId);

3
engine/storage/cache/src/main/java/org/apache/cloudstack/storage/cache/manager/StorageCacheManagerImpl.java vendored
View File

@ -32,6 +32,7 @@ import java.util.concurrent.TimeUnit;
import javax.inject.Inject;
import javax.naming.ConfigurationException;
import com.cloud.configuration.ConfigurationManager;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
@ -161,7 +162,7 @@ public class StorageCacheManagerImpl implements StorageCacheManager, Manager {
public boolean configure(String name, Map<String, Object> params) throws ConfigurationException {
cacheReplacementEnabled = Boolean.parseBoolean(configDao.getValue(Config.StorageCacheReplacementEnabled.key()));
cacheReplaceMentInterval = NumbersUtil.parseInt(configDao.getValue(Config.StorageCacheReplacementInterval.key()), 86400);
workers = NumbersUtil.parseInt(configDao.getValue(Config.ExpungeWorkers.key()), 10);
workers = ConfigurationManager.ExpungeWorkers.value();
executors = Executors.newScheduledThreadPool(workers, new NamedThreadFactory("StorageCacheManager-cache-replacement"));
return true;
}

12
plugins/storage/image/s3/src/main/java/org/apache/cloudstack/storage/datastore/driver/S3ImageStoreDriverImpl.java
View File

@ -25,6 +25,7 @@ import java.util.Map;
import javax.inject.Inject;
import com.cloud.configuration.ConfigurationManager;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
@ -37,7 +38,6 @@ import com.cloud.agent.api.to.DataStoreTO;
import com.cloud.agent.api.to.S3TO;
import com.cloud.configuration.Config;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.utils.NumbersUtil;
import com.cloud.utils.storage.S3.S3Utils;
public class S3ImageStoreDriverImpl extends BaseImageStoreDriverImpl {
@ -86,22 +86,18 @@ public class S3ImageStoreDriverImpl extends BaseImageStoreDriverImpl {
*/
S3TO s3 = (S3TO)getStoreTO(store);
if(logger.isDebugEnabled()) {
logger.debug("Generating pre-signed s3 entity extraction URL for object: " + key);
}
logger.debug("Generating pre-signed s3 entity extraction URL for object: {}", key);
Date expiration = new Date();
long milliSeconds = expiration.getTime();
// Get extract url expiration interval set in global configuration (in seconds)
String urlExpirationInterval = _configDao.getValue(Config.ExtractURLExpirationInterval.toString());
// Expired after configured interval (in milliseconds), default 14400 seconds
milliSeconds += 1000 * NumbersUtil.parseInt(urlExpirationInterval, 14400);
milliSeconds += 1000L * ConfigurationManager.ExtractURLExpirationInterval.value();
expiration.setTime(milliSeconds);
URL s3url = S3Utils.generatePresignedUrl(s3, s3.getBucketName(), key, expiration);
logger.info("Pre-Signed URL = " + s3url.toString());
logger.info("Pre-Signed URL = {}", s3url.toString());
return s3url.toString();
}

19
plugins/storage/image/swift/src/main/java/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java
View File

@ -24,6 +24,7 @@ import java.util.UUID;
import javax.inject.Inject;
import com.cloud.configuration.ConfigurationManager;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult;
import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
@ -33,7 +34,6 @@ import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector;
import org.apache.cloudstack.engine.subsystem.api.storage.StorageCacheManager;
import org.apache.cloudstack.framework.async.AsyncCallbackDispatcher;
import org.apache.cloudstack.framework.async.AsyncCompletionCallback;
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.apache.cloudstack.storage.command.DownloadCommand;
import org.apache.cloudstack.storage.datastore.db.ImageStoreDetailsDao;
import org.apache.cloudstack.storage.image.BaseImageStoreDriverImpl;
@ -44,7 +44,6 @@ import com.cloud.agent.api.storage.DownloadAnswer;
import com.cloud.agent.api.to.DataObjectType;
import com.cloud.agent.api.to.DataStoreTO;
import com.cloud.agent.api.to.SwiftTO;
import com.cloud.configuration.Config;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.utils.SwiftUtil;
import com.cloud.utils.exception.CloudRuntimeException;
@ -57,8 +56,6 @@ public class SwiftImageStoreDriverImpl extends BaseImageStoreDriverImpl {
EndPointSelector _epSelector;
@Inject
StorageCacheManager cacheManager;
@Inject
ConfigurationDao _configDao;
@Override
public DataStoreTO getStoreTO(DataStore store) {
@ -83,15 +80,11 @@ public class SwiftImageStoreDriverImpl extends BaseImageStoreDriverImpl {
String containerName = SwiftUtil.getContainerName(dataObject.getType().toString(), dataObject.getId());
String objectName = installPath.split("\\/")[1];
// Get extract url expiration interval set in global configuration (in seconds)
int urlExpirationInterval = Integer.parseInt(_configDao.getValue(Config.ExtractURLExpirationInterval.toString()));
int urlExpirationInterval = ConfigurationManager.ExtractURLExpirationInterval.value();
URL swiftUrl = SwiftUtil.generateTempUrl(swiftTO, containerName, objectName, tempKey, urlExpirationInterval);
if (swiftUrl != null) {
logger.debug("Swift temp-url: " + swiftUrl.toString());
return swiftUrl.toString();
}
throw new CloudRuntimeException("Unable to create extraction URL");
logger.debug("Swift temp-url: {}", swiftUrl);
return swiftUrl.toString();
}
@Override
@ -115,7 +108,7 @@ public class SwiftImageStoreDriverImpl extends BaseImageStoreDriverImpl {
throw new CloudRuntimeException(errMsg);
}
CreateContext<CreateCmdResult> context = new CreateContext<CreateCmdResult>(callback, data);
CreateContext<CreateCmdResult> context = new CreateContext<>(callback, data);
AsyncCallbackDispatcher<SwiftImageStoreDriverImpl, DownloadAnswer> caller = AsyncCallbackDispatcher.create(this);
caller.setContext(context);
@ -125,7 +118,5 @@ public class SwiftImageStoreDriverImpl extends BaseImageStoreDriverImpl {
caller.setCallback(caller.getTarget().createVolumeAsyncCallback(null, null));
}
ep.sendMessageAsync(dcmd, caller);
}
}

65
server/src/main/java/com/cloud/configuration/Config.java
View File

@ -424,31 +424,6 @@ public enum Config {
"The interval (in seconds) between cleanup for removed accounts",
null),
InstanceName("Advanced", AgentManager.class, String.class, "instance.name", "VM", "Name of the deployment instance.", "instanceName"),
ExpungeDelay(
"Advanced",
UserVmManager.class,
Integer.class,
"expunge.delay",
"86400",
"Determines how long (in seconds) to wait before actually expunging destroyed vm. The default value = the default value of expunge.interval",
null),
ExpungeInterval(
"Advanced",
UserVmManager.class,
Integer.class,
"expunge.interval",
"86400",
"The interval (in seconds) to wait before running the expunge thread.",
null),
ExpungeWorkers("Advanced", UserVmManager.class, Integer.class, "expunge.workers", "1", "Number of workers performing expunge ", null),
ExtractURLCleanUpInterval(
"Advanced",
ManagementServer.class,
Integer.class,
"extract.url.cleanup.interval",
"7200",
"The interval (in seconds) to wait before cleaning up the extract URL's ",
null),
DisableExtraction(
"Advanced",
ManagementServer.class,
@ -457,14 +432,6 @@ public enum Config {
"false",
"Flag for disabling extraction of Templates, ISOs, Snapshots and volumes",
null),
ExtractURLExpirationInterval(
"Advanced",
ManagementServer.class,
Integer.class,
"extract.url.expiration.interval",
"14400",
"The life of an extract URL after which it is deleted ",
null),
HostStatsInterval(
"Advanced",
ManagementServer.class,
@ -750,38 +717,6 @@ public enum Config {
"3600",
"Time (in seconds) to wait before taking over a VM in transition state",
null),
VmDiskThrottlingIopsReadRate(
"Advanced",
ManagementServer.class,
Integer.class,
"vm.disk.throttling.iops_read_rate",
"0",
"Default disk I/O read rate in requests per second allowed in User vm's disk.",
null),
VmDiskThrottlingIopsWriteRate(
"Advanced",
ManagementServer.class,
Integer.class,
"vm.disk.throttling.iops_write_rate",
"0",
"Default disk I/O writerate in requests per second allowed in User vm's disk.",
null),
VmDiskThrottlingBytesReadRate(
"Advanced",
ManagementServer.class,
Integer.class,
"vm.disk.throttling.bytes_read_rate",
"0",
"Default disk I/O read rate in bytes per second allowed in User vm's disk.",
null),
VmDiskThrottlingBytesWriteRate(
"Advanced",
ManagementServer.class,
Integer.class,
"vm.disk.throttling.bytes_write_rate",
"0",
"Default disk I/O writerate in bytes per second allowed in User vm's disk.",
null),
ControlCidr(
"Advanced",
ManagementServer.class,

3
server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java
View File

@ -8489,7 +8489,8 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
BYTES_MAX_READ_LENGTH, BYTES_MAX_WRITE_LENGTH, ADD_HOST_ON_SERVICE_RESTART_KVM, SET_HOST_DOWN_TO_MAINTENANCE,
VM_SERVICE_OFFERING_MAX_CPU_CORES, VM_SERVICE_OFFERING_MAX_RAM_SIZE, MIGRATE_VM_ACROSS_CLUSTERS,
ENABLE_ACCOUNT_SETTINGS_FOR_DOMAIN, ENABLE_DOMAIN_SETTINGS_FOR_CHILD_DOMAIN,
ALLOW_DOMAIN_ADMINS_TO_CREATE_TAGGED_OFFERINGS, DELETE_QUERY_BATCH_SIZE, AllowNonRFC1918CompliantIPs, HostCapacityTypeCpuMemoryWeight
ALLOW_DOMAIN_ADMINS_TO_CREATE_TAGGED_OFFERINGS, DELETE_QUERY_BATCH_SIZE, AllowNonRFC1918CompliantIPs, HostCapacityTypeCpuMemoryWeight,
ExpungeDelay, ExpungeInterval, ExpungeWorkers, ExtractURLCleanUpInterval, ExtractURLExpirationInterval
};
}

592
server/src/main/java/com/cloud/storage/StorageManagerImpl.java
View File

File diff suppressed because it is too large Load Diff