etan39
/
cloudstack
mirror of https://github.com/apache/cloudstack.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Update ECS Plugin: address reviewer comments for ECS Plugin

This commit is contained in:
MHK 2025-12-28 14:19:31 +03:00
parent a10e06e161
commit 6019139fef
6 changed files with 946 additions and 703 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
plugins/storage/object/ECS/src/main/java/org/apache/cloudstack/storage/datastore/driver/EcsConstants.java Normal file
View File

@ -0,0 +1,19 @@
package org.apache.cloudstack.storage.datastore.driver;
public final class EcsConstants {
private EcsConstants() {}
// Object store details keys
public static final String MGMT_URL = "mgmt_url";
public static final String SA_USER = "sa_user";
public static final String SA_PASS = "sa_password";
public static final String NAMESPACE = "namespace";
public static final String INSECURE = "insecure";
public static final String S3_HOST = "s3_host";
public static final String USER_PREFIX = "user_prefix";
public static final String DEFAULT_USER_PREFIX = "cs-";
// Per-account keys
public static final String AD_KEY_ACCESS = "ecs.accesskey";
public static final String AD_KEY_SECRET = "ecs.secretkey";
}

142
plugins/storage/object/ECS/src/main/java/org/apache/cloudstack/storage/datastore/driver/EcsMgmtTokenManager.java Normal file
View File

@ -0,0 +1,142 @@
package org.apache.cloudstack.storage.datastore.driver;
import java.nio.charset.StandardCharsets;
import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.auth.BasicScheme;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.util.EntityUtils;
import com.cloud.utils.exception.CloudRuntimeException;
public class EcsMgmtTokenManager {
private static final long DEFAULT_TOKEN_MAX_AGE_SEC = 300;
private static final long EXPIRY_SKEW_SEC = 30;
private static final ConcurrentHashMap<TokenKey, TokenEntry> TOKEN_CACHE = new ConcurrentHashMap<>();
private static final ConcurrentHashMap<TokenKey, Object> TOKEN_LOCKS = new ConcurrentHashMap<>();
static final class EcsUnauthorizedException extends RuntimeException {
EcsUnauthorizedException(final String msg) { super(msg); }
}
@FunctionalInterface
public interface WithToken<T> { T run(String token) throws Exception; }
private static final class TokenKey {
final String mgmtUrl;
final String user;
TokenKey(final String mgmtUrl, final String user) {
this.mgmtUrl = mgmtUrl;
this.user = user;
}
@Override public boolean equals(final Object o) {
if (this == o) return true;
if (!(o instanceof TokenKey)) return false;
final TokenKey k = (TokenKey) o;
return Objects.equals(mgmtUrl, k.mgmtUrl) && Objects.equals(user, k.user);
}
@Override public int hashCode() { return Objects.hash(mgmtUrl, user); }
}
private static final class TokenEntry {
final String token;
final long expiresAtMs;
TokenEntry(final String token, final long expiresAtMs) {
this.token = token;
this.expiresAtMs = expiresAtMs;
}
boolean validNow() {
return token != null && !token.isBlank() && System.currentTimeMillis() < expiresAtMs;
}
}
public <T> T callWithRetry401(final EcsObjectStoreDriverImpl.EcsCfg cfg,
final WithToken<T> op,
final HttpClientFactory httpFactory) throws Exception {
try {
return op.run(getAuthToken(cfg, httpFactory));
} catch (EcsUnauthorizedException u) {
invalidate(cfg);
return op.run(getAuthToken(cfg, httpFactory));
}
}
public void invalidate(final EcsObjectStoreDriverImpl.EcsCfg cfg) {
TOKEN_CACHE.remove(new TokenKey(trimTail(cfg.mgmtUrl), cfg.saUser));
}
public String getAuthToken(final EcsObjectStoreDriverImpl.EcsCfg cfg,
final HttpClientFactory httpFactory) {
final String mu = trimTail(cfg.mgmtUrl);
final TokenKey key = new TokenKey(mu, cfg.saUser);
final TokenEntry cached = TOKEN_CACHE.get(key);
if (cached != null && cached.validNow()) return cached.token;
final Object lock = TOKEN_LOCKS.computeIfAbsent(key, k -> new Object());
synchronized (lock) {
final TokenEntry cached2 = TOKEN_CACHE.get(key);
if (cached2 != null && cached2.validNow()) return cached2.token;
final TokenEntry fresh = loginAndGetTokenFresh(mu, cfg.saUser, cfg.saPass, cfg.insecure, httpFactory);
TOKEN_CACHE.put(key, fresh);
return fresh.token;
}
}
private TokenEntry loginAndGetTokenFresh(final String mgmtUrl,
final String user,
final String pass,
final boolean insecure,
final HttpClientFactory httpFactory) {
try (CloseableHttpClient http = httpFactory.build(insecure)) {
final HttpGet get = new HttpGet(mgmtUrl + "/login");
UsernamePasswordCredentials creds = new UsernamePasswordCredentials(user, pass);
get.addHeader(new BasicScheme().authenticate(creds, get, null));
try (CloseableHttpResponse resp = http.execute(get)) {
final int status = resp.getStatusLine().getStatusCode();
if (status != 200 && status != 201) {
final String body = resp.getEntity() != null
? EntityUtils.toString(resp.getEntity(), StandardCharsets.UTF_8)
: "";
throw new CloudRuntimeException("ECS /login failed: HTTP " + status + " body=" + body);
}
if (resp.getFirstHeader("X-SDS-AUTH-TOKEN") == null) {
throw new CloudRuntimeException("ECS /login did not return X-SDS-AUTH-TOKEN header");
}
final String token = resp.getFirstHeader("X-SDS-AUTH-TOKEN").getValue();
long maxAgeSec = DEFAULT_TOKEN_MAX_AGE_SEC;
try {
if (resp.getFirstHeader("X-SDS-AUTH-MAX-AGE") != null) {
maxAgeSec = Long.parseLong(resp.getFirstHeader("X-SDS-AUTH-MAX-AGE").getValue().trim());
}
} catch (Exception ignore) { }
final long effectiveSec = Math.max(5, maxAgeSec - EXPIRY_SKEW_SEC);
final long expiresAtMs = System.currentTimeMillis() + (effectiveSec * 1000L);
return new TokenEntry(token, expiresAtMs);
}
} catch (Exception e) {
throw new CloudRuntimeException("Failed to obtain ECS auth token: " + e.getMessage(), e);
}
}
private static String trimTail(final String s) {
if (s == null) return null;
return s.endsWith("/") ? s.substring(0, s.length() - 1) : s;
}
/** Simple seam for testability; implemented by the driver using its existing buildHttpClient(). */
@FunctionalInterface
public interface HttpClientFactory {
CloseableHttpClient build(boolean insecure);
}
}

1334
plugins/storage/object/ECS/src/main/java/org/apache/cloudstack/storage/datastore/driver/EcsObjectStoreDriverImpl.java
View File

File diff suppressed because it is too large Load Diff

71
plugins/storage/object/ECS/src/main/java/org/apache/cloudstack/storage/datastore/driver/EcsXmlParser.java Normal file
View File

@ -0,0 +1,71 @@
package org.apache.cloudstack.storage.datastore.driver;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
public class EcsXmlParser {
public Integer parseIntTag(final String xml, final String tag) {
String v = extractTag(xml, tag);
if (v == null) return null;
try { return Integer.parseInt(v.trim()); } catch (NumberFormatException ignore) { return null; }
}
public String extractTag(final String xml, final String tag) {
if (xml == null) return null;
final String open = "<" + tag + ">";
final String close = "</" + tag + ">";
int i = xml.indexOf(open);
if (i < 0) return null;
int j = xml.indexOf(close, i + open.length());
if (j < 0) return null;
return xml.substring(i + open.length(), j).trim();
}
public List<String> extractAllTags(final String xml, final String tag) {
final List<String> out = new ArrayList<>();
if (xml == null) return out;
final String open = "<" + tag + ">";
final String close = "</" + tag + ">";
int from = 0;
while (true) {
int i = xml.indexOf(open, from);
if (i < 0) break;
int j = xml.indexOf(close, i + open.length());
if (j < 0) break;
out.add(xml.substring(i + open.length(), j).trim());
from = j + close.length();
}
return out;
}
public void extractKeysFromListBucketXml(final String xml, final List<String> keys) {
if (xml == null) return;
final String contentsOpen = "<Contents>";
final String contentsClose = "</Contents>";
int from = 0;
while (true) {
int i = xml.indexOf(contentsOpen, from);
if (i < 0) break;
int j = xml.indexOf(contentsClose, i + contentsOpen.length());
if (j < 0) break;
String block = xml.substring(i, j + contentsClose.length());
String key = extractTag(block, "Key");
if (key != null && !key.isEmpty()) keys.add(key.trim());
from = j + contentsClose.length();
}
}
public boolean looksLikeBucketAlreadyExists400(final String respBody) {
final String lb = respBody == null ? "" : respBody.toLowerCase(Locale.ROOT);
return lb.contains("already exist")
|| lb.contains("already_exists")
|| lb.contains("already-exists")
|| lb.contains("name already in use")
|| lb.contains("bucket exists")
|| lb.contains("duplicate");
}
}

44
plugins/storage/object/ECS/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/EcsObjectStoreLifeCycleImpl.java
View File

@ -29,6 +29,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.HostScope;
import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope;
import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO;
import org.apache.cloudstack.storage.datastore.driver.EcsConstants;
import org.apache.cloudstack.storage.object.datastore.ObjectStoreHelper;
import org.apache.cloudstack.storage.object.datastore.ObjectStoreProviderManager;
import org.apache.cloudstack.storage.object.store.lifecycle.ObjectStoreLifeCycle;
@ -36,7 +37,7 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpGet; // change to POST if ECS needs it
import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.impl.auth.BasicScheme;
import org.apache.http.impl.client.CloseableHttpClient;
@ -52,16 +53,6 @@ public class EcsObjectStoreLifeCycleImpl implements ObjectStoreLifeCycle {
private static final Logger LOG = LogManager.getLogger(EcsObjectStoreLifeCycleImpl.class);
// detail keys coming from the API
private static final String MGMT_URL = "mgmt_url";
private static final String SA_USER = "sa_user";
private static final String SA_PASS = "sa_password";
private static final String INSECURE = "insecure";
// optional details (currently not used in persistence logic but accepted)
private static final String S3_HOST = "s3_host";
private static final String NAMESPACE = "namespace";
private static final String PROVIDER_NAME = "ECS";
@Inject
@ -79,19 +70,21 @@ public class EcsObjectStoreLifeCycleImpl implements ObjectStoreLifeCycle {
final String url = getString(dsInfos, "url", true);
final String name = getString(dsInfos, "name", true);
final Long size = getLong(dsInfos, "size");
final Long size = getLong(dsInfos, "size"); // optional
final String providerName = getProviderName(dsInfos);
final Map<String, String> details = getDetails(dsInfos);
final Map<String, String> details = getDetails(dsInfos); // typed map, no unchecked cast
final EcsConfig cfg = verifyAndNormalize(details);
LOG.info("ECS initialize: provider='{}', name='{}', url='{}', mgmt_url='{}', insecure={}, s3_host='{}', namespace='{}'",
providerName, name, url, cfg.mgmtUrl, cfg.insecure,
details.get(S3_HOST), details.get(NAMESPACE));
details.get(EcsConstants.S3_HOST), details.get(EcsConstants.NAMESPACE));
// Try ECS login up-front so we fail fast on bad config
loginAndGetToken(cfg.mgmtUrl, cfg.saUser, cfg.saPass, cfg.insecure);
// Put “canonical” values back into details (so DB keeps what we validated)
applyCanonicalDetails(details, cfg);
final Map<String, Object> objectStoreParameters = buildObjectStoreParams(name, url, size, providerName);
@ -228,14 +221,14 @@ public class EcsObjectStoreLifeCycleImpl implements ObjectStoreLifeCycle {
}
private static EcsConfig verifyAndNormalize(final Map<String, String> details) {
final String mgmtUrl = trim(details.get(MGMT_URL));
final String saUser = safe(details.get(SA_USER));
final String saPass = safe(details.get(SA_PASS));
final boolean insecure = Boolean.parseBoolean(details.getOrDefault(INSECURE, "false"));
final String mgmtUrl = trim(details.get(EcsConstants.MGMT_URL));
final String saUser = safe(details.get(EcsConstants.SA_USER));
final String saPass = safe(details.get(EcsConstants.SA_PASS));
final boolean insecure = Boolean.parseBoolean(details.getOrDefault(EcsConstants.INSECURE, "false"));
verifyRequiredDetail(MGMT_URL, mgmtUrl);
verifyRequiredDetail(SA_USER, saUser);
verifyRequiredDetail(SA_PASS, saPass);
verifyRequiredDetail(EcsConstants.MGMT_URL, mgmtUrl);
verifyRequiredDetail(EcsConstants.SA_USER, saUser);
verifyRequiredDetail(EcsConstants.SA_PASS, saPass);
return new EcsConfig(mgmtUrl, saUser, saPass, insecure);
}
@ -247,10 +240,11 @@ public class EcsObjectStoreLifeCycleImpl implements ObjectStoreLifeCycle {
}
private static void applyCanonicalDetails(final Map<String, String> details, final EcsConfig cfg) {
details.put(MGMT_URL, cfg.mgmtUrl);
details.put(SA_USER, cfg.saUser);
details.put(SA_PASS, cfg.saPass);
details.put(INSECURE, Boolean.toString(cfg.insecure));
details.put(EcsConstants.MGMT_URL, cfg.mgmtUrl);
details.put(EcsConstants.SA_USER, cfg.saUser);
details.put(EcsConstants.SA_PASS, cfg.saPass);
details.put(EcsConstants.INSECURE, Boolean.toString(cfg.insecure));
// keep any optional keys already present (S3_HOST, NAMESPACE, etc.)
}
private static Map<String, Object> buildObjectStoreParams(final String name,

39
ui/src/views/infra/AddObjectStorage.vue
View File

@ -97,9 +97,9 @@
<!-- S3 host (hostname[:port], no scheme) -> details[1].value (s3_host) -->
<a-form-item name="s3Host" ref="s3Host" :rules="[{ required: true, message: $t('label.required') }]">
<template #label>
<tooltip-label :title="'ECS Private URL'" :tooltip="'The internal S3 endpoint URL used by CloudStack to communicate with ECS.May be the same as the Public URL.'"/>
<tooltip-label :title="'ECS Private URL'" :tooltip="'The internal S3 endpoint URL used by CloudStack to communicate with ECS. May be the same as the Public URL.'"/>
</template>
<a-input v-model:value="form.s3Host" placeholder="https://ecs.example.com" />
<a-input v-model:value="form.s3Host" placeholder="ecs.example.com or ecs.example.com:9020" />
</a-form-item>
<!-- Service account user -> details[2].value (sa_user) -->
@ -126,7 +126,15 @@
<a-input v-model:value="form.namespace" placeholder="cloudstack" />
</a-form-item>
<!-- Insecure -> details[5].value (insecure) -->
<!-- User prefix -> details[5].value (user_prefix) -->
<a-form-item name="userPrefix" ref="userPrefix">
<template #label>
<tooltip-label :title="'User prefix'" :tooltip="'Prefix used for ECS user creation. Default is cs- (user_prefix). Example: cs-'"/>
</template>
<a-input v-model:value="form.userPrefix" placeholder="cs-" />
</a-form-item>
<!-- Insecure -> details[6].value (insecure) -->
<a-form-item name="insecure" ref="insecure">
<a-checkbox v-model:checked="form.insecure">
Allow insecure HTTPS (set insecure=true)
@ -210,6 +218,7 @@ export default {
mgmtUrl: '',
s3Host: '',
namespace: '',
userPrefix: 'cs-',
insecure: false
})
this.rules = reactive({
@ -252,14 +261,8 @@ export default {
data['details[4].key'] = 'iamUrl'
data['details[4].value'] = values.iamUrl
} else if (provider === 'ECS') {
// ECS details matching your CLI example:
// add objectstoragepool name=ecsstore-4 provider=ECS url=https://ecs.earthlink.dev
// details[0].key=mgmt_url details[0].value=https://ecs-api.elcld.net
// details[1].key=s3_host details[1].value=ecs.earthlink.dev
// details[2].key=sa_user details[2].value=cloudstack
// details[3].key=sa_password details[3].value=...
// details[4].key=namespace details[4].value=cloudstack
// details[5].key=insecure details[5].value=true
// ECS details:
// details[0]=mgmt_url, [1]=s3_host, [2]=sa_user, [3]=sa_password, [4]=namespace, [5]=user_prefix, [6]=insecure
data['details[0].key'] = 'mgmt_url'
data['details[0].value'] = values.mgmtUrl
@ -276,8 +279,18 @@ export default {
data['details[4].key'] = 'namespace'
data['details[4].value'] = values.namespace
data['details[5].key'] = 'insecure'
data['details[5].value'] = values.insecure ? 'true' : 'false'
// Optional; only send if user entered something (driver defaults to cs- when missing)
if (values.userPrefix && values.userPrefix.trim() !== '') {
data['details[5].key'] = 'user_prefix'
data['details[5].value'] = values.userPrefix.trim()
} else {
// keep ordering stable for insecure when prefix omitted
data['details[5].key'] = 'user_prefix'
data['details[5].value'] = 'cs-'
}
data['details[6].key'] = 'insecure'
data['details[6].value'] = values.insecure ? 'true' : 'false'
} else {
// Generic non-Cloudian, non-ECS object stores
data['details[0].key'] = 'accesskey'