etan39
/
cloudstack
mirror of https://github.com/apache/cloudstack.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch '4.22'

This commit is contained in:
Suresh Kumar Anaparti 2026-03-04 21:13:01 +05:30
parent 9cc6c09b9e 65e54091ff
commit 75620309c2
No known key found for this signature in database
GPG Key ID: D7CEAE3A9E71D0AA
19 changed files with 225 additions and 122 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
api/src/main/java/org/apache/cloudstack/api/ApiServerService.java
View File

@ -49,5 +49,7 @@ public interface ApiServerService {
boolean resetPassword(UserAccount userAccount, String token, String password);
String getDomainId(Map<String, Object[]> params);
boolean isPostRequestsAndTimestampsEnforced();
}

7
engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java
View File

@ -846,15 +846,18 @@ public class VMInstanceDaoImpl extends GenericDaoBase<VMInstanceVO, Long> implem
try {
pstmtLegacy = txn.prepareAutoCloseStatement(finalQueryLegacy.toString());
pstmt = txn.prepareAutoCloseStatement(finalQuery.toString());
for (int i = 0; i < resourceIdList.size(); i++) {
pstmtLegacy.setLong(1 + i, resourceIdList.get(i));
pstmt.setLong(1 + i, resourceIdList.get(i));
}
ResultSet rs = pstmtLegacy.executeQuery();
while (rs.next()) {
result.put(rs.getString(1).concat(rs.getString(2)), rs.getLong(3));
}
pstmt = txn.prepareAutoCloseStatement(finalQuery.toString());
for (int i = 0; i < resourceIdList.size(); i++) {
pstmt.setLong(1 + i, resourceIdList.get(i));
}
rs = pstmt.executeQuery();
while (rs.next()) {
result.put(rs.getString(1).concat(rs.getString(2)), rs.getLong(3));

3
engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java
View File

@ -2565,8 +2565,7 @@ public class StorageSystemDataMotionStrategy implements DataMotionStrategy {
throw new CloudRuntimeException("Destination storage pool with ID " + dataStore.getId() + " was not located.");
}
boolean isSrcAndDestPoolPowerFlexStorage = srcStoragePoolVO.getPoolType().equals(Storage.StoragePoolType.PowerFlex) && destStoragePoolVO.getPoolType().equals(Storage.StoragePoolType.PowerFlex);
if (srcStoragePoolVO.isManaged() && !isSrcAndDestPoolPowerFlexStorage && srcStoragePoolVO.getId() != destStoragePoolVO.getId()) {
if (srcStoragePoolVO.isManaged() && srcStoragePoolVO.getId() != destStoragePoolVO.getId()) {
throw new CloudRuntimeException("Migrating a volume online with KVM from managed storage is not currently supported.");
}

8
framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java
View File

@ -420,6 +420,14 @@ public class ConfigKey<T> {
return value();
}
/**
* @deprecated
* Still used by some external code, but use {@link ConfigKey#valueInScope(Scope, Long)} instead.
*/
public T valueInDomain(Long domainId) {
return valueInScope(Scope.Domain, domainId);
}
public T valueInScope(Scope scope, Long id) {
if (id == null) {
return value();

14
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java
View File

@ -884,7 +884,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
protected StorageSubsystemCommandHandler storageHandler;
private boolean convertInstanceVerboseMode = false;
private String[] convertInstanceEnv = null;
private Map<String, String> convertInstanceEnv = null;
protected boolean dpdkSupport = false;
protected String dpdkOvsPath;
protected String directDownloadTemporaryDownloadPath;
@ -949,7 +949,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
return convertInstanceVerboseMode;
}
public String[] getConvertInstanceEnv() {
public Map<String, String> getConvertInstanceEnv() {
return convertInstanceEnv;
}
@ -1439,14 +1439,14 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
return;
}
if (StringUtils.isNotBlank(convertEnvTmpDir) && StringUtils.isNotBlank(convertEnvVirtv2vTmpDir)) {
convertInstanceEnv = new String[2];
convertInstanceEnv[0] = String.format("%s=%s", "TMPDIR", convertEnvTmpDir);
convertInstanceEnv[1] = String.format("%s=%s", "VIRT_V2V_TMPDIR", convertEnvVirtv2vTmpDir);
convertInstanceEnv = new HashMap<>(2);
convertInstanceEnv.put("TMPDIR", convertEnvTmpDir);
convertInstanceEnv.put("VIRT_V2V_TMPDIR", convertEnvVirtv2vTmpDir);
} else {
convertInstanceEnv = new String[1];
convertInstanceEnv = new HashMap<>(1);
String key = StringUtils.isNotBlank(convertEnvTmpDir) ? "TMPDIR" : "VIRT_V2V_TMPDIR";
String value = StringUtils.isNotBlank(convertEnvTmpDir) ? convertEnvTmpDir : convertEnvVirtv2vTmpDir;
convertInstanceEnv[0] = String.format("%s=%s", key, value);
convertInstanceEnv.put(key, value);
}
}

9
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapper.java
View File

@ -22,9 +22,11 @@ import java.net.URLEncoder;
import java.nio.charset.Charset;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang3.StringUtils;
import com.cloud.agent.api.Answer;
@ -244,7 +246,12 @@ public class LibvirtConvertInstanceCommandWrapper extends CommandWrapper<Convert
String logPrefix = String.format("(%s) virt-v2v ovf source: %s progress", originalVMName, sourceOVFDirPath);
OutputInterpreter.LineByLineOutputLogger outputLogger = new OutputInterpreter.LineByLineOutputLogger(logger, logPrefix);
script.execute(outputLogger);
Map<String, String> convertInstanceEnv = serverResource.getConvertInstanceEnv();
if (MapUtils.isEmpty(convertInstanceEnv)) {
script.execute(outputLogger);
} else {
script.execute(outputLogger, convertInstanceEnv);
}
int exitValue = script.getExitValue();
return exitValue == 0;
}

7
plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptorTest.java
View File

@ -180,7 +180,8 @@ public class ScaleIOStorageAdaptorTest {
details.put(ScaleIOGatewayClient.STORAGE_POOL_MDMS, "1.1.1.1,2.2.2.2");
when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl status scini"))).thenReturn(3);
when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl is-enabled scini"))).thenReturn(0);
when(Script.runSimpleBashScript(Mockito.eq("/opt/emc/scaleio/sdc/bin/drv_cfg --query_mdms|grep 1.1.1.1"))).thenReturn("MDM-ID 71fd458f0775010f SDC ID 4421a91a00000000 INSTALLATION ID 204930df2cbcaf8e IPs [0]-3.3.3.3 [1]-4.4.4.4");
when(Script.runSimpleBashScript(Mockito.eq("/opt/emc/scaleio/sdc/bin/drv_cfg --query_mdms|grep 1.1.1.1"), Mockito.eq(ScaleIOUtil.DEFAULT_TIMEOUT_MS)))
.thenReturn("MDM-ID 71fd458f0775010f SDC ID 4421a91a00000000 INSTALLATION ID 204930df2cbcaf8e IPs [0]-3.3.3.3 [1]-4.4.4.4");
Pair<Boolean, String> result = scaleIOStorageAdaptor.unprepareStorageClient(poolUuid, details);
@ -196,11 +197,11 @@ public class ScaleIOStorageAdaptorTest {
when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl is-enabled scini"))).thenReturn(0);
when(Script.executeCommand(Mockito.eq("sed -i '/1.1.1.1\\,/d' /etc/emc/scaleio/drv_cfg.txt"))).thenReturn(new Pair<>(null, null));
when(Script.runSimpleBashScriptForExitValue(Mockito.eq("systemctl restart scini"))).thenReturn(0);
when(Script.runSimpleBashScript(Mockito.eq("/opt/emc/scaleio/sdc/bin/drv_cfg --query_mdms|grep 1.1.1.1"))).thenReturn("MDM-ID 71fd458f0775010f SDC ID 4421a91a00000000 INSTALLATION ID 204930df2cbcaf8e IPs [0]-1.1.1.1 [1]-2.2.2.2");
when(Script.runSimpleBashScript(Mockito.eq("/opt/emc/scaleio/sdc/bin/drv_cfg --query_mdms --file /etc/emc/scaleio/drv_cfg.txt|grep 1.1.1.1"), Mockito.eq(ScaleIOUtil.DEFAULT_TIMEOUT_MS)))
.thenReturn("MDM-ID 71fd458f0775010f SDC ID 4421a91a00000000 INSTALLATION ID 204930df2cbcaf8e IPs [0]-1.1.1.1 [1]-2.2.2.2");
when(Script.executeCommand(Mockito.eq("/opt/emc/scaleio/sdc/bin/drv_cfg"))).thenReturn(new Pair<>(null, null));
when(Script.executeCommand(Mockito.eq("/opt/emc/scaleio/sdc/bin/drv_cfg --query_vols"))).thenReturn(new Pair<>("", null));
Pair<Boolean, String> result = scaleIOStorageAdaptor.unprepareStorageClient(poolUuid, details);
Assert.assertFalse(result.first());

13
plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/ScaleIOStoragePoolTest.java
View File

@ -95,9 +95,8 @@ public class ScaleIOStoragePoolTest {
details.put(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID, systemId);
try (MockedStatic<Script> ignored = Mockito.mockStatic(Script.class)) {
when(Script.runSimpleBashScript(
"/opt/emc/scaleio/sdc/bin/drv_cfg --query_mdms|grep 218ce1797566a00f|awk '{print $5}'")).thenReturn(
sdcId);
when(Script.runSimpleBashScript(Mockito.eq("/opt/emc/scaleio/sdc/bin/drv_cfg --query_mdms --file /etc/emc/scaleio/drv_cfg.txt|grep 218ce1797566a00f|awk '{print $5}'"), Mockito.eq(ScaleIOUtil.DEFAULT_TIMEOUT_MS)))
.thenReturn(sdcId);
ScaleIOStoragePool pool1 = new ScaleIOStoragePool(uuid, "192.168.1.19", 443, "a519be2f00000000", type,
details, adapter);
@ -116,10 +115,10 @@ public class ScaleIOStoragePoolTest {
details.put(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID, systemId);
try (MockedStatic<Script> ignored = Mockito.mockStatic(Script.class)) {
when(Script.runSimpleBashScript(
"/opt/emc/scaleio/sdc/bin/drv_cfg --query_mdms|grep 218ce1797566a00f|awk '{print $5}'")).thenReturn(
null);
when(Script.runSimpleBashScript("/opt/emc/scaleio/sdc/bin/drv_cfg --query_guid")).thenReturn(sdcGuid);
when(Script.runSimpleBashScript(Mockito.eq("/opt/emc/scaleio/sdc/bin/drv_cfg --query_mdms --file /etc/emc/scaleio/drv_cfg.txt|grep 218ce1797566a00f|awk '{print $5}'"), Mockito.eq(ScaleIOUtil.DEFAULT_TIMEOUT_MS)))
.thenReturn(null);
when(Script.runSimpleBashScript(Mockito.eq("/opt/emc/scaleio/sdc/bin/drv_cfg --query_guid --file /etc/emc/scaleio/drv_cfg.txt"), Mockito.eq(ScaleIOUtil.DEFAULT_TIMEOUT_MS)))
.thenReturn(sdcGuid);
ScaleIOStoragePool pool1 = new ScaleIOStoragePool(uuid, "192.168.1.19", 443, "a519be2f00000000", type,
details, adapter);

89
plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClientConnectionPool.java
View File

@ -17,12 +17,12 @@
package org.apache.cloudstack.storage.datastore.client;
import java.net.URISyntaxException;
import java.security.KeyManagementException;
import java.security.NoSuchAlgorithmException;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;
import com.cloud.storage.StoragePool;
import com.cloud.utils.exception.CloudRuntimeException;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailVO;
import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao;
@ -36,9 +36,9 @@ import com.google.common.base.Preconditions;
public class ScaleIOGatewayClientConnectionPool {
protected Logger logger = LogManager.getLogger(getClass());
private ConcurrentHashMap<Long, ScaleIOGatewayClient> gatewayClients;
private Map<Long, ScaleIOGatewayClient> gatewayClients;
private static final ScaleIOGatewayClientConnectionPool instance;
private final Object lock = new Object();
static {
instance = new ScaleIOGatewayClientConnectionPool();
@ -49,60 +49,67 @@ public class ScaleIOGatewayClientConnectionPool {
}
private ScaleIOGatewayClientConnectionPool() {
gatewayClients = new ConcurrentHashMap<Long, ScaleIOGatewayClient>();
gatewayClients = new ConcurrentHashMap<>();
}
public ScaleIOGatewayClient getClient(StoragePool storagePool,
StoragePoolDetailsDao storagePoolDetailsDao)
throws NoSuchAlgorithmException, KeyManagementException, URISyntaxException {
StoragePoolDetailsDao storagePoolDetailsDao) {
return getClient(storagePool.getId(), storagePool.getUuid(), storagePoolDetailsDao);
}
public ScaleIOGatewayClient getClient(DataStore dataStore,
StoragePoolDetailsDao storagePoolDetailsDao)
throws NoSuchAlgorithmException, KeyManagementException, URISyntaxException {
StoragePoolDetailsDao storagePoolDetailsDao) {
return getClient(dataStore.getId(), dataStore.getUuid(), storagePoolDetailsDao);
}
private ScaleIOGatewayClient getClient(Long storagePoolId, String storagePoolUuid,
StoragePoolDetailsDao storagePoolDetailsDao)
throws NoSuchAlgorithmException, KeyManagementException, URISyntaxException {
StoragePoolDetailsDao storagePoolDetailsDao) {
Preconditions.checkArgument(storagePoolId != null && storagePoolId > 0,
"Invalid storage pool id");
ScaleIOGatewayClient client = null;
synchronized (gatewayClients) {
client = gatewayClients.get(storagePoolId);
if (client == null) {
String url = null;
StoragePoolDetailVO urlDetail = storagePoolDetailsDao.findDetail(storagePoolId, ScaleIOGatewayClient.GATEWAY_API_ENDPOINT);
if (urlDetail != null) {
url = urlDetail.getValue();
}
String username = null;
StoragePoolDetailVO encryptedUsernameDetail = storagePoolDetailsDao.findDetail(storagePoolId, ScaleIOGatewayClient.GATEWAY_API_USERNAME);
if (encryptedUsernameDetail != null) {
final String encryptedUsername = encryptedUsernameDetail.getValue();
username = DBEncryptionUtil.decrypt(encryptedUsername);
}
String password = null;
StoragePoolDetailVO encryptedPasswordDetail = storagePoolDetailsDao.findDetail(storagePoolId, ScaleIOGatewayClient.GATEWAY_API_PASSWORD);
if (encryptedPasswordDetail != null) {
final String encryptedPassword = encryptedPasswordDetail.getValue();
password = DBEncryptionUtil.decrypt(encryptedPassword);
}
final int clientTimeout = StorageManager.STORAGE_POOL_CLIENT_TIMEOUT.valueIn(storagePoolId);
final int clientMaxConnections = StorageManager.STORAGE_POOL_CLIENT_MAX_CONNECTIONS.valueIn(storagePoolId);
logger.debug("Getting ScaleIO client for {} ({})", storagePoolId, storagePoolUuid);
client = new ScaleIOGatewayClientImpl(url, username, password, false, clientTimeout, clientMaxConnections);
gatewayClients.put(storagePoolId, client);
logger.debug("Added gateway client for the storage pool [id: {}, uuid: {}]", storagePoolId, storagePoolUuid);
ScaleIOGatewayClient client = gatewayClients.get(storagePoolId);
if (client == null) {
logger.debug("Before acquiring lock to create ScaleIO client for {} ({})", storagePoolId, storagePoolUuid);
synchronized (lock) {
logger.debug("Acquired lock to create ScaleIO client for {} ({})", storagePoolId, storagePoolUuid);
client = gatewayClients.get(storagePoolId);
if (client == null) {
logger.debug("Initializing ScaleIO client for {} ({})", storagePoolId, storagePoolUuid);
String url = Optional.ofNullable(storagePoolDetailsDao.findDetail(storagePoolId, ScaleIOGatewayClient.GATEWAY_API_ENDPOINT))
.map(StoragePoolDetailVO::getValue)
.orElse(null);
String username = Optional.ofNullable(storagePoolDetailsDao.findDetail(storagePoolId, ScaleIOGatewayClient.GATEWAY_API_USERNAME))
.map(StoragePoolDetailVO::getValue)
.map(DBEncryptionUtil::decrypt)
.orElse(null);
String password = Optional.ofNullable(storagePoolDetailsDao.findDetail(storagePoolId, ScaleIOGatewayClient.GATEWAY_API_PASSWORD))
.map(StoragePoolDetailVO::getValue)
.map(DBEncryptionUtil::decrypt)
.orElse(null);
int clientTimeout = StorageManager.STORAGE_POOL_CLIENT_TIMEOUT.valueIn(storagePoolId);
int clientMaxConnections = StorageManager.STORAGE_POOL_CLIENT_MAX_CONNECTIONS.valueIn(storagePoolId);
try {
client = new ScaleIOGatewayClientImpl(url, username, password, false, clientTimeout, clientMaxConnections);
logger.debug("Created ScaleIO client for the storage pool [id: {}, uuid: {}]", storagePoolId, storagePoolUuid);
gatewayClients.put(storagePoolId, client);
} catch (Exception e) {
String msg = String.format("Failed to create ScaleIO client for the storage pool [id: %d, uuid: %s]", storagePoolId, storagePoolUuid);
throw new CloudRuntimeException(msg, e);
}
}
}
}
logger.debug("Returning ScaleIO client for {} ({})", storagePoolId, storagePoolUuid);
return client;
}
@ -110,8 +117,8 @@ public class ScaleIOGatewayClientConnectionPool {
Preconditions.checkArgument(dataStore != null && dataStore.getId() > 0,
"Invalid storage pool id");
ScaleIOGatewayClient client = null;
synchronized (gatewayClients) {
ScaleIOGatewayClient client;
synchronized (lock) {
client = gatewayClients.remove(dataStore.getId());
}

22
plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClientImpl.java
View File

@ -92,7 +92,7 @@ public class ScaleIOGatewayClientImpl implements ScaleIOGatewayClient {
private String username;
private String password;
private String sessionKey = null;
private String sessionKey;
// The session token is valid for 8 hours from the time it was created, unless there has been no activity for 10 minutes
// Reference: https://cpsdocs.dellemc.com/bundle/PF_REST_API_RG/page/GUID-92430F19-9F44-42B6-B898-87D5307AE59B.html
@ -102,7 +102,7 @@ public class ScaleIOGatewayClientImpl implements ScaleIOGatewayClient {
private static final long MAX_IDLE_TIME_IN_MILLISECS = MAX_IDLE_TIME_IN_MINS * 60 * 1000;
private static final long BUFFER_TIME_IN_MILLISECS = 30 * 1000; // keep 30 secs buffer before the expiration (to avoid any last-minute operations)
private boolean authenticating = false;
private volatile boolean authenticating = false;
private long createTime = 0;
private long lastUsedTime = 0;
@ -142,7 +142,6 @@ public class ScaleIOGatewayClientImpl implements ScaleIOGatewayClient {
this.username = username;
this.password = password;
authenticate();
logger.debug("API client for the PowerFlex gateway " + apiURI.getHost() + " is created successfully, with max connections: "
+ maxConnections + " and timeout: " + timeout + " secs");
}
@ -181,7 +180,7 @@ public class ScaleIOGatewayClientImpl implements ScaleIOGatewayClient {
long now = System.currentTimeMillis();
createTime = lastUsedTime = now;
} catch (final IOException e) {
logger.error("Failed to authenticate PowerFlex API Gateway " + apiURI.getHost() + " due to: " + e.getMessage() + getConnectionManagerStats());
logger.error("Failed to authenticate PowerFlex API Gateway " + apiURI.getHost() + " due to: " + e.getMessage() + getConnectionManagerStats(), e);
throw new CloudRuntimeException("Failed to authenticate PowerFlex API Gateway " + apiURI.getHost() + " due to: " + e.getMessage());
} finally {
authenticating = false;
@ -199,6 +198,10 @@ public class ScaleIOGatewayClientImpl implements ScaleIOGatewayClient {
}
private boolean isSessionExpired() {
if (sessionKey == null) {
logger.debug("Session never created for the Gateway " + apiURI.getHost());
return true;
}
long now = System.currentTimeMillis() + BUFFER_TIME_IN_MILLISECS;
if ((now - createTime) > MAX_VALID_SESSION_TIME_IN_MILLISECS) {
logger.debug("Session expired for the Gateway " + apiURI.getHost() + ", token is invalid after " + MAX_VALID_SESSION_TIME_IN_HRS
@ -281,7 +284,11 @@ public class ScaleIOGatewayClientImpl implements ScaleIOGatewayClient {
HttpResponse response = null;
boolean responseConsumed = false;
try {
while (authenticating); // wait for authentication request (if any) to complete (and to pick the new session key)
while (authenticating) { // wait for authentication request (if any)
// to complete (and to pick the new session key)
Thread.yield();
}
final HttpGet request = new HttpGet(apiURI.toString() + path);
request.setHeader(HttpHeaders.AUTHORIZATION, "Basic " + Base64.getEncoder().encodeToString((this.username + ":" + this.sessionKey).getBytes()));
logger.debug("Sending GET request: " + request.toString());
@ -316,7 +323,10 @@ public class ScaleIOGatewayClientImpl implements ScaleIOGatewayClient {
HttpResponse response = null;
boolean responseConsumed = false;
try {
while (authenticating); // wait for authentication request (if any) to complete (and to pick the new session key)
while (authenticating) { // wait for authentication request (if any)
// to complete (and to pick the new session key)
Thread.yield();
}
final HttpPost request = new HttpPost(apiURI.toString() + path);
request.setHeader(HttpHeaders.AUTHORIZATION, "Basic " + Base64.getEncoder().encodeToString((this.username + ":" + this.sessionKey).getBytes()));
request.setHeader("content-type", "application/json");

10
plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/driver/ScaleIOPrimaryDataStoreDriver.java
View File

@ -153,15 +153,15 @@ public class ScaleIOPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
sdcManager = new ScaleIOSDCManagerImpl();
}
ScaleIOGatewayClient getScaleIOClient(final StoragePool storagePool) throws Exception {
ScaleIOGatewayClient getScaleIOClient(final StoragePool storagePool) {
return ScaleIOGatewayClientConnectionPool.getInstance().getClient(storagePool, storagePoolDetailsDao);
}
ScaleIOGatewayClient getScaleIOClient(final DataStore dataStore) throws Exception {
ScaleIOGatewayClient getScaleIOClient(final DataStore dataStore) {
return ScaleIOGatewayClientConnectionPool.getInstance().getClient(dataStore, storagePoolDetailsDao);
}
private boolean setVolumeLimitsOnSDC(VolumeVO volume, Host host, DataStore dataStore, Long iopsLimit, Long bandwidthLimitInKbps) throws Exception {
private boolean setVolumeLimitsOnSDC(VolumeVO volume, Host host, DataStore dataStore, Long iopsLimit, Long bandwidthLimitInKbps) {
sdcManager = ComponentContext.inject(sdcManager);
final String sdcId = sdcManager.prepareSDC(host, dataStore);
if (StringUtils.isBlank(sdcId)) {
@ -173,7 +173,7 @@ public class ScaleIOPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
return client.mapVolumeToSdcWithLimits(ScaleIOUtil.getVolumePath(volume.getPath()), sdcId, iopsLimit, bandwidthLimitInKbps);
}
private boolean setVolumeLimitsFromDetails(VolumeVO volume, Host host, DataStore dataStore) throws Exception {
private boolean setVolumeLimitsFromDetails(VolumeVO volume, Host host, DataStore dataStore) {
Long bandwidthLimitInKbps = 0L; // Unlimited
// Check Bandwidth Limit parameter in volume details
final VolumeDetailVO bandwidthVolumeDetail = volumeDetailsDao.findDetail(volume.getId(), Volume.BANDWIDTH_LIMIT_IN_MBPS);
@ -997,7 +997,7 @@ public class ScaleIOPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
return host;
}
public void updateSnapshotsAfterCopyVolume(DataObject srcData, DataObject destData) throws Exception {
public void updateSnapshotsAfterCopyVolume(DataObject srcData, DataObject destData) {
final long srcVolumeId = srcData.getId();
DataStore srcStore = srcData.getDataStore();
final ScaleIOGatewayClient client = getScaleIOClient(srcStore);

32
plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/manager/ScaleIOSDCManagerImpl.java
View File

@ -361,14 +361,13 @@ public class ScaleIOSDCManagerImpl implements ScaleIOSDCManager, Configurable {
return false;
}
try {
if (logger.isDebugEnabled()) {
List<StoragePoolHostVO> poolHostVOsBySdc = storagePoolHostDao.findByLocalPath(sdcId);
if (CollectionUtils.isNotEmpty(poolHostVOsBySdc) && poolHostVOsBySdc.size() > 1) {
logger.debug(String.format("There are other connected pools with the same SDC of the host %s", host));
}
}
List<StoragePoolHostVO> poolHostVOsBySdc = storagePoolHostDao.findByLocalPath(sdcId);
if (CollectionUtils.isNotEmpty(poolHostVOsBySdc) && poolHostVOsBySdc.size() > 1) {
logger.debug(String.format("There are other connected pools with the same SDC of the host %s", host));
return false;
}
try {
return !areVolumesMappedToPoolSdc(dataStore.getId(), sdcId);
} catch (Exception e) {
logger.warn("Unable to check whether the SDC of the pool: " + dataStore.getId() + " can be unprepared on the host: " + host.getId() + ", due to " + e.getMessage(), e);
@ -425,20 +424,26 @@ public class ScaleIOSDCManagerImpl implements ScaleIOSDCManager, Configurable {
@Override
public boolean isHostSdcConnected(String sdcId, DataStore dataStore, int waitTimeInSecs) {
long poolId = dataStore.getId();
logger.debug(String.format("Waiting (for %d secs) for the SDC %s of the pool %s to connect",
waitTimeInSecs, sdcId, dataStore));
logger.debug("Waiting (for {} secs) for the SDC {} of the pool {} to connect", waitTimeInSecs, sdcId, dataStore);
int timeBetweenTries = 1000; // Try more frequently (every sec) and return early if connected
while (waitTimeInSecs > 0) {
for (int i = 0; i < waitTimeInSecs; i++) {
logger.debug("Attempt {} of {} for the SDC {} of the pool {} to connect", i + 1, waitTimeInSecs, sdcId, dataStore);
if (isHostSdcConnected(sdcId, poolId)) {
logger.debug("Attempt {} of {} successful for the SDC {} of the pool {} to connect", i + 1, waitTimeInSecs, sdcId, dataStore);
return true;
}
waitTimeInSecs--;
try {
Thread.sleep(timeBetweenTries);
} catch (Exception ignore) {
}
}
return isHostSdcConnected(sdcId, poolId);
boolean isConnected = isHostSdcConnected(sdcId, poolId);
if (isConnected) {
logger.debug("Final attempt to connect the SDC {} of the pool {} succeeded", sdcId, dataStore);
} else {
logger.debug("Final attempt to connect the SDC {} of the pool {} failed", sdcId, dataStore);
}
return isConnected;
}
@Override
@ -470,6 +475,7 @@ public class ScaleIOSDCManagerImpl implements ScaleIOSDCManager, Configurable {
private boolean isHostSdcConnected(String sdcId, long poolId) {
try {
final ScaleIOGatewayClient client = getScaleIOClient(poolId);
logger.debug("Checking whether SDC {} connected or not", sdcId);
return client.isSdcConnected(sdcId);
} catch (Exception e) {
logger.error("Failed to check host SDC connection", e);
@ -477,7 +483,7 @@ public class ScaleIOSDCManagerImpl implements ScaleIOSDCManager, Configurable {
}
}
private ScaleIOGatewayClient getScaleIOClient(final Long storagePoolId) throws Exception {
private ScaleIOGatewayClient getScaleIOClient(Long storagePoolId) {
StoragePoolVO storagePool = storagePoolDao.findById(storagePoolId);
if (storagePool == null) {
throw new CloudRuntimeException("Unable to find the storage pool with id " + storagePoolId);

35
plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/util/ScaleIOUtil.java
View File

@ -158,6 +158,11 @@ public class ScaleIOUtil {
*/
private static final Pattern DRV_CFG_MDM_IPS_PATTERN = Pattern.compile("\\s*,\\s*");
/**
* Default command execution timeout 5 minutes.
*/
public static final int DEFAULT_TIMEOUT_MS = 5 * 60 * 1000;
public static boolean addMdms(String... mdmAddresses) {
if (mdmAddresses.length < 1) {
return false;
@ -226,14 +231,10 @@ public class ScaleIOUtil {
}
} else {
String command = String.format(REMOVE_MDM_CMD_TEMPLATE, mdmAddress, DRV_CFG_FILE);
String stdErr = Script.executeCommand(command).second();
if(StringUtils.isEmpty(stdErr)) {
// restart SDC needed only if configuration file modified manually (not by CLI)
restartSDC = true;
changesApplied = true;
} else {
LOGGER.error(String.format("Failed to remove MDM %s from %s: %s", mdmAddress, DRV_CFG_FILE, stdErr));
}
Script.runSimpleBashScript(command, DEFAULT_TIMEOUT_MS);
// restart SDC needed only if configuration file modified manually (not by CLI)
restartSDC = true;
changesApplied = true;
}
}
if (restartSDC) {
@ -338,9 +339,9 @@ public class ScaleIOUtil {
*/
public static boolean isMdmPresent(String mdmAddress) {
//query_mdms outputs "MDM-ID <System/MDM-Id> SDC ID <SDC-Id> INSTALLATION ID <Installation-Id> IPs [0]-x.x.x.x [1]-x.x.x.x" for a MDM with ID: <MDM-Id>
String queryMdmsCmd = ScaleIOUtil.SDC_HOME_PATH + "/bin/" + ScaleIOUtil.QUERY_MDMS_CMD;
String queryMdmsCmd = ScaleIOUtil.SDC_HOME_PATH + "/bin/" + ScaleIOUtil.QUERY_MDMS_CMD + " --file " + DRV_CFG_FILE;
queryMdmsCmd += "|grep " + mdmAddress;
String result = Script.runSimpleBashScript(queryMdmsCmd);
String result = Script.runSimpleBashScript(queryMdmsCmd, DEFAULT_TIMEOUT_MS);
return StringUtils.isNotBlank(result) && result.contains(mdmAddress);
}
@ -348,7 +349,7 @@ public class ScaleIOUtil {
public static String getSdcHomePath() {
String sdcHomePropertyCmdFormat = "sed -n '/%s/p' '%s' 2>/dev/null | sed 's/%s=//g' 2>/dev/null";
String sdcHomeCmd = String.format(sdcHomePropertyCmdFormat, SDC_HOME_PARAMETER, AGENT_PROPERTIES_FILE, SDC_HOME_PARAMETER);
String result = Script.runSimpleBashScript(sdcHomeCmd);
String result = Script.runSimpleBashScript(sdcHomeCmd, DEFAULT_TIMEOUT_MS);
String sdcHomePath;
if (result == null) {
sdcHomePath = DEFAULT_SDC_HOME_PATH;
@ -364,7 +365,7 @@ public class ScaleIOUtil {
// Detecting new volumes
String rescanCmd = ScaleIOUtil.SDC_HOME_PATH + "/bin/" + ScaleIOUtil.RESCAN_CMD;
String result = Script.runSimpleBashScript(rescanCmd);
String result = Script.runSimpleBashScript(rescanCmd, DEFAULT_TIMEOUT_MS);
if (result == null) {
LOGGER.warn("Failed to rescan for new volumes");
}
@ -375,7 +376,7 @@ public class ScaleIOUtil {
String queryDiskCmd = SDC_HOME_PATH + "/bin/" + ScaleIOUtil.QUERY_VOLUMES_CMD;
queryDiskCmd += "|grep " + volumeId + "|awk '{print $4}'";
String result = Script.runSimpleBashScript(queryDiskCmd);
String result = Script.runSimpleBashScript(queryDiskCmd, DEFAULT_TIMEOUT_MS);
if (result == null) {
LOGGER.warn("Query volumes failed to get volume: " + volumeId + " details for system id");
return null;
@ -390,8 +391,8 @@ public class ScaleIOUtil {
}
public static String getSdcGuid() {
String queryGuidCmd = ScaleIOUtil.SDC_HOME_PATH + "/bin/" + ScaleIOUtil.QUERY_GUID_CMD;
String result = Script.runSimpleBashScript(queryGuidCmd);
String queryGuidCmd = ScaleIOUtil.SDC_HOME_PATH + "/bin/" + ScaleIOUtil.QUERY_GUID_CMD + " --file " + DRV_CFG_FILE;
String result = Script.runSimpleBashScript(queryGuidCmd, DEFAULT_TIMEOUT_MS);
if (result == null) {
LOGGER.warn("Failed to get SDC guid");
return null;
@ -412,9 +413,9 @@ public class ScaleIOUtil {
public static String getSdcId(String mdmId) {
//query_mdms outputs "MDM-ID <System/MDM-Id> SDC ID <SDC-Id> INSTALLATION ID <Installation-Id> IPs [0]-x.x.x.x [1]-x.x.x.x" for a MDM with ID: <MDM-Id>
String queryMdmsCmd = ScaleIOUtil.SDC_HOME_PATH + "/bin/" + ScaleIOUtil.QUERY_MDMS_CMD;
String queryMdmsCmd = ScaleIOUtil.SDC_HOME_PATH + "/bin/" + ScaleIOUtil.QUERY_MDMS_CMD + " --file " + DRV_CFG_FILE;
queryMdmsCmd += "|grep " + mdmId + "|awk '{print $5}'";
String result = Script.runSimpleBashScript(queryMdmsCmd);
String result = Script.runSimpleBashScript(queryMdmsCmd, DEFAULT_TIMEOUT_MS);
if (result == null) {
LOGGER.warn("Failed to get SDC Id, for the MDM: " + mdmId);
return null;

21
plugins/storage/volume/scaleio/src/test/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClientImplTest.java
View File

@ -99,8 +99,6 @@ public class ScaleIOGatewayClientImplTest {
@Test
public void testClientAuthSuccess() {
Assert.assertNotNull(client);
wireMockRule.verify(getRequestedFor(urlEqualTo("/api/login"))
.withBasicAuth(new BasicCredentials(username, password)));
wireMockRule.stubFor(get("/api/types/StoragePool/instances")
.willReturn(aResponse()
@ -110,25 +108,28 @@ public class ScaleIOGatewayClientImplTest {
client.listStoragePools();
wireMockRule.verify(getRequestedFor(urlEqualTo("/api/login"))
.withBasicAuth(new BasicCredentials(username, password)));
wireMockRule.verify(getRequestedFor(urlEqualTo("/api/types/StoragePool/instances"))
.withBasicAuth(new BasicCredentials(username, sessionKey)));
}
@Test(expected = CloudRuntimeException.class)
public void testClientAuthFailure() throws Exception {
Assert.assertNotNull(client);
wireMockRule.stubFor(get("/api/login")
.willReturn(unauthorized()
.withHeader("content-type", "application/json;charset=UTF-8")
.withBody("")));
new ScaleIOGatewayClientImpl("https://localhost/api", username, password, false, timeout, maxConnections);
client.listStoragePools();
}
@Test(expected = ServerApiException.class)
public void testRequestTimeout() {
Assert.assertNotNull(client);
wireMockRule.verify(getRequestedFor(urlEqualTo("/api/login"))
.withBasicAuth(new BasicCredentials(username, password)));
wireMockRule.stubFor(get("/api/types/StoragePool/instances")
.willReturn(aResponse()
@ -143,14 +144,15 @@ public class ScaleIOGatewayClientImplTest {
@Test
public void testCreateSingleVolume() {
Assert.assertNotNull(client);
wireMockRule.verify(getRequestedFor(urlEqualTo("/api/login"))
.withBasicAuth(new BasicCredentials(username, password)));
final String volumeName = "testvolume";
final String scaleIOStoragePoolId = "4daaa55e00000000";
final int sizeInGb = 8;
Volume scaleIOVolume = client.createVolume(volumeName, scaleIOStoragePoolId, sizeInGb, Storage.ProvisioningType.THIN);
wireMockRule.verify(getRequestedFor(urlEqualTo("/api/login"))
.withBasicAuth(new BasicCredentials(username, password)));
wireMockRule.verify(postRequestedFor(urlEqualTo("/api/types/Volume/instances"))
.withBasicAuth(new BasicCredentials(username, sessionKey))
.withRequestBody(containing("\"name\":\"" + volumeName + "\""))
@ -169,8 +171,6 @@ public class ScaleIOGatewayClientImplTest {
@Test
public void testCreateMultipleVolumes() {
Assert.assertNotNull(client);
wireMockRule.verify(getRequestedFor(urlEqualTo("/api/login"))
.withBasicAuth(new BasicCredentials(username, password)));
final String volumeNamePrefix = "testvolume_";
final String scaleIOStoragePoolId = "4daaa55e00000000";
@ -188,6 +188,9 @@ public class ScaleIOGatewayClientImplTest {
Assert.assertEquals(scaleIOVolume.getVolumeType(), Volume.VolumeType.ThinProvisioned);
}
wireMockRule.verify(getRequestedFor(urlEqualTo("/api/login"))
.withBasicAuth(new BasicCredentials(username, password)));
wireMockRule.verify(volumesCount, postRequestedFor(urlEqualTo("/api/types/Volume/instances"))
.withBasicAuth(new BasicCredentials(username, sessionKey))
.withRequestBody(containing("\"name\":\"" + volumeNamePrefix))

8
plugins/user-authenticators/oauth2/src/main/java/org/apache/cloudstack/oauth2/api/command/OauthLoginAPIAuthenticatorCmd.java
View File

@ -183,14 +183,14 @@ public class OauthLoginAPIAuthenticatorCmd extends BaseCmd implements APIAuthent
}
protected Long getDomainIdFromParams(Map<String, Object[]> params, StringBuilder auditTrailSb, String responseType) {
String[] domainIdArr = (String[])params.get(ApiConstants.DOMAIN_ID);
String domainIdStr = _apiServer.getDomainId(params);
Long domainId = null;
if (domainIdArr != null && domainIdArr.length > 0) {
if (StringUtils.isNotEmpty(domainIdStr)) {
try {
//check if UUID is passed in for domain
domainId = _apiServer.fetchDomainId(domainIdArr[0]);
domainId = _apiServer.fetchDomainId(domainIdStr);
if (domainId == null) {
domainId = Long.parseLong(domainIdArr[0]);
domainId = Long.parseLong(domainIdStr);
}
auditTrailSb.append(" domainid=" + domainId);// building the params for POST call
} catch (final NumberFormatException e) {

19
plugins/user-authenticators/oauth2/src/test/java/org/apache/cloudstack/oauth2/api/command/OauthLoginAPIAuthenticatorCmdTest.java
View File

@ -85,10 +85,29 @@ public class OauthLoginAPIAuthenticatorCmdTest {
ApiServer apiServer = mock(ApiServer.class);
cmd._apiServer = apiServer;
when(apiServer.fetchDomainId("1234")).thenReturn(5678L);
when(apiServer.getDomainId(params)).thenCallRealMethod();
Long domainId = cmd.getDomainIdFromParams(params, auditTrailSb, responseType);
assertEquals(Long.valueOf(5678), domainId);
assertEquals(" domainid=5678", auditTrailSb.toString());
}
@Test
public void testGetDomainIdFromCamelCaseParam() {
StringBuilder auditTrailSb = new StringBuilder();
String responseType = "json";
Map<String, Object[]> params = new HashMap<>();
params.put(ApiConstants.DOMAIN_ID, null);
params.put(ApiConstants.DOMAIN__ID, new String[]{"5678"});
ApiServer apiServer = mock(ApiServer.class);
cmd._apiServer = apiServer;
when(apiServer.fetchDomainId("5678")).thenReturn(1234L);
when(apiServer.getDomainId(params)).thenCallRealMethod();
Long domainId = cmd.getDomainIdFromParams(params, auditTrailSb, responseType);
assertEquals(Long.valueOf(1234), domainId);
assertEquals(" domainid=1234", auditTrailSb.toString());
}
}

19
server/src/main/java/com/cloud/api/ApiServer.java
View File

@ -1413,6 +1413,25 @@ public class ApiServer extends ManagerBase implements HttpRequestHandler, ApiSer
return userPasswordResetManager.validateAndResetPassword(userAccount, token, password);
}
@Override
public String getDomainId(Map<String, Object[]> params) {
if (MapUtils.isEmpty(params)) {
return null;
}
String[] domainIdArr = (String[])params.get(ApiConstants.DOMAIN_ID);
if (domainIdArr == null) {
// Fallback to support clients using the camelCase parameter name "domainId"
domainIdArr = (String[])params.get(ApiConstants.DOMAIN__ID);
}
if (domainIdArr == null || domainIdArr.length == 0) {
return null;
}
return domainIdArr[0];
}
private void checkCommandAvailable(final User user, final String commandName, final InetAddress remoteAddress) throws PermissionDeniedException {
if (user == null) {
throw new PermissionDeniedException("User is null for role based API access check for command" + commandName);

9
server/src/main/java/com/cloud/api/auth/DefaultLoginAPIAuthenticatorCmd.java
View File

@ -20,6 +20,7 @@ import com.cloud.api.ApiServlet;
import com.cloud.domain.Domain;
import com.cloud.user.User;
import com.cloud.user.UserAccount;
import com.cloud.utils.StringUtils;
import org.apache.cloudstack.api.ApiServerService;
import com.cloud.api.response.ApiResponseSerializer;
import com.cloud.exception.CloudAuthenticationException;
@ -115,14 +116,14 @@ public class DefaultLoginAPIAuthenticatorCmd extends BaseCmd implements APIAuthe
// FIXME: ported from ApiServlet, refactor and cleanup
final String[] username = (String[])params.get(ApiConstants.USERNAME);
final String[] password = (String[])params.get(ApiConstants.PASSWORD);
final String[] domainIdArr = (String[])params.get(ApiConstants.DOMAIN_ID);
String domainIdStr = _apiServer.getDomainId(params);
Long domainId = null;
if (domainIdArr != null && domainIdArr.length > 0) {
if (StringUtils.isNotEmpty(domainIdStr)) {
try {
//check if UUID is passed in for domain
domainId = _apiServer.fetchDomainId(domainIdArr[0]);
domainId = _apiServer.fetchDomainId(domainIdStr);
if (domainId == null) {
domainId = Long.parseLong(domainIdArr[0]);
domainId = Long.parseLong(domainIdStr);
}
auditTrailSb.append(" domainid=" + domainId);// building the params for POST call
} catch (final NumberFormatException e) {

20
utils/src/main/java/com/cloud/utils/script/Script.java
View File

@ -31,6 +31,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.Callable;
@ -44,6 +45,7 @@ import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.io.IOUtils;
import org.apache.logging.log4j.LogManager;
@ -243,6 +245,14 @@ public class Script implements Callable<String> {
}
public String execute(OutputInterpreter interpreter) {
return executeInternal(interpreter, null);
}
public String execute(OutputInterpreter interpreter, Map<String, String> environment) {
return executeInternal(interpreter, environment);
}
private String executeInternal(OutputInterpreter interpreter, Map<String, String> environment) {
String[] command = _command.toArray(new String[_command.size()]);
String commandLine = buildCommandLine(command);
if (_logger.isDebugEnabled() ) {
@ -254,11 +264,19 @@ public class Script implements Callable<String> {
ProcessBuilder pb = new ProcessBuilder(command);
pb.redirectErrorStream(true);
if (_workDir != null)
if (MapUtils.isNotEmpty(environment)) {
Map<String, String> processEnvironment = pb.environment();
processEnvironment.putAll(environment);
}
if (_workDir != null) {
pb.directory(new File(_workDir));
}
_logger.trace(String.format("Starting process for command [%s].", commandLine));
_process = pb.start();
if (_process == null) {
_logger.warn(String.format("Unable to execute command [%s] because no process was created.", commandLine));
return "Unable to execute the command: " + command[0];