etan39
/
cloudstack
mirror of https://github.com/apache/cloudstack.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Update provision certificate command to allow provisioning via SSH

This commit is contained in:
vishesh92 2026-03-25 19:13:03 +05:30
parent 1679d20266
commit bc7f95c863
No known key found for this signature in database
GPG Key ID: 4E395186CBFA790B
8 changed files with 201 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
api/src/main/java/org/apache/cloudstack/api/command/admin/ca/ProvisionCertificateCmd.java
View File

@ -63,6 +63,12 @@ public class ProvisionCertificateCmd extends BaseAsyncCmd {
description = "Name of the CA service provider, otherwise the default configured provider plugin will be used")
private String provider;
@Parameter(name = ApiConstants.FORCED, type = CommandType.BOOLEAN,
description = "When true, uses SSH to re-provision the agent's certificate, bypassing the NIO agent connection. " +
"Use this when agents are disconnected due to a CA change. Supported for KVM hosts and SystemVMs. Default is false",
since = "4.23.0")
private Boolean forced;
/////////////////////////////////////////////////////
/////////////////// Accessors ///////////////////////
/////////////////////////////////////////////////////
@ -79,6 +85,10 @@ public class ProvisionCertificateCmd extends BaseAsyncCmd {
return provider;
}
public boolean isForced() {
return forced != null && forced;
}
/////////////////////////////////////////////////////
/////////////// API Implementation///////////////////
/////////////////////////////////////////////////////
@ -90,7 +100,7 @@ public class ProvisionCertificateCmd extends BaseAsyncCmd {
throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Unable to find host by ID: " + getHostId());
}
boolean result = caManager.provisionCertificate(host, getReconnect(), getProvider());
boolean result = caManager.provisionCertificate(host, getReconnect(), getProvider(), isForced());
SuccessResponse response = new SuccessResponse(getCommandName());
response.setSuccess(result);
setResponseObject(response);

19
api/src/main/java/org/apache/cloudstack/ca/CAManager.java
View File

@ -23,6 +23,8 @@ import java.security.cert.X509Certificate;
import java.util.List;
import java.util.Map;
import com.trilead.ssh2.Connection;
import org.apache.cloudstack.framework.ca.CAProvider;
import org.apache.cloudstack.framework.ca.CAService;
import org.apache.cloudstack.framework.ca.Certificate;
@ -139,12 +141,25 @@ public interface CAManager extends CAService, Configurable, PluggableService {
boolean revokeCertificate(final BigInteger certSerial, final String certCn, final String provider);
/**
* Provisions certificate for given active and connected agent host
* Provisions certificate for given agent host.
* When forced=true, uses SSH to re-provision bypassing the NIO agent connection (for disconnected agents).
* @param host
* @param reconnect
* @param provider
* @param forced when true, provisions via SSH instead of NIO; supports KVM hosts and SystemVMs
* @return returns success/failure as boolean
*/
boolean provisionCertificate(final Host host, final Boolean reconnect, final String provider);
boolean provisionCertificate(final Host host, final Boolean reconnect, final String provider, final boolean forced);
/**
* Provisions certificate for a KVM host using an existing SSH connection.
* Runs keystore-setup to generate a CSR, issues a certificate, then runs keystore-cert-import.
* Used during host discovery and for forced re-provisioning when the NIO agent is unreachable.
* @param sshConnection active SSH connection to the KVM host
* @param agentIp IP address of the KVM host agent
* @param agentHostname hostname of the KVM host agent
*/
void provisionCertificateViaSsh(Connection sshConnection, String agentIp, String agentHostname);
/**
* Setups up a new keystore and generates CSR for a host

6
scripts/util/keystore-cert-import
View File

@ -70,8 +70,8 @@ elif [ ! -f "$CACERT_FILE" ]; then
fi
# Import cacerts into the keystore
awk '/-----BEGIN CERTIFICATE-----?/{n++}{print > "cloudca." n }' "$CACERT_FILE"
for caChain in $(ls cloudca.*); do
awk 'BEGIN{n=0} /-----BEGIN CERTIFICATE-----/{n++}{print > "cloudca." n }' "$CACERT_FILE"
for caChain in $(ls cloudca.* 2>/dev/null); do
keytool -delete -noprompt -alias "$caChain" -keystore "$KS_FILE" -storepass "$KS_PASS" > /dev/null 2>&1 || true
keytool -import -noprompt -storepass "$KS_PASS" -trustcacerts -alias "$caChain" -file "$caChain" -keystore "$KS_FILE" > /dev/null 2>&1
done
@ -143,7 +143,7 @@ if [ -f "$SYSTEM_FILE" ]; then
REALHOSTIP_KS_FILE="$(dirname $(dirname $PROPS_FILE))/certs/realhostip.keystore"
REALHOSTIP_PASS="vmops.com"
if [ -f "$REALHOSTIP_KS_FILE" ]; then
awk '/-----BEGIN CERTIFICATE-----/{n++}{print > "cloudca." n }' "$CACERT_FILE"
awk 'BEGIN{n=0} /-----BEGIN CERTIFICATE-----/{n++}{print > "cloudca." n }' "$CACERT_FILE"
for caChain in $(ls cloudca.* 2>/dev/null); do
keytool -delete -noprompt -alias "$caChain" -keystore "$REALHOSTIP_KS_FILE" \
-storepass "$REALHOSTIP_PASS" > /dev/null 2>&1 || true

55
server/src/main/java/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java
View File

@ -21,7 +21,6 @@ import static com.cloud.configuration.ConfigurationManagerImpl.ADD_HOST_ON_SERVI
import java.net.InetAddress;
import java.net.URI;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@ -32,11 +31,8 @@ import javax.naming.ConfigurationException;
import org.apache.cloudstack.agent.lb.IndirectAgentLB;
import org.apache.cloudstack.ca.CAManager;
import org.apache.cloudstack.ca.SetupCertificateCommand;
import org.apache.cloudstack.direct.download.DirectDownloadManager;
import org.apache.cloudstack.framework.ca.Certificate;
import org.apache.cloudstack.utils.cache.LazyCache;
import org.apache.cloudstack.utils.security.KeyStoreUtils;
import com.cloud.agent.AgentManager;
import com.cloud.agent.Listener;
@ -66,7 +62,6 @@ import com.cloud.resource.DiscovererBase;
import com.cloud.resource.ResourceStateAdapter;
import com.cloud.resource.ServerResource;
import com.cloud.resource.UnableDeleteHostException;
import com.cloud.utils.PasswordGenerator;
import com.cloud.utils.StringUtils;
import com.cloud.utils.UuidUtils;
import com.cloud.utils.exception.CloudRuntimeException;
@ -174,55 +169,7 @@ public abstract class LibvirtServerDiscoverer extends DiscovererBase implements
throw new CloudRuntimeException("Cannot secure agent communication because SSH connection is invalid for host IP=" + agentIp);
}
Integer validityPeriod = CAManager.CertValidityPeriod.value();
if (validityPeriod < 1) {
validityPeriod = 1;
}
String keystorePassword = PasswordGenerator.generateRandomPassword(16);
final SSHCmdHelper.SSHCmdResult keystoreSetupResult = SSHCmdHelper.sshExecuteCmdWithResult(sshConnection,
String.format("sudo /usr/share/cloudstack-common/scripts/util/%s " +
"/etc/cloudstack/agent/agent.properties " +
"/etc/cloudstack/agent/%s " +
"%s %d " +
"/etc/cloudstack/agent/%s",
KeyStoreUtils.KS_SETUP_SCRIPT,
KeyStoreUtils.KS_FILENAME,
keystorePassword,
validityPeriod,
KeyStoreUtils.CSR_FILENAME));
if (!keystoreSetupResult.isSuccess()) {
throw new CloudRuntimeException("Failed to setup keystore on the KVM host: " + agentIp);
}
final Certificate certificate = caManager.issueCertificate(keystoreSetupResult.getStdOut(), Arrays.asList(agentHostname, agentIp), Collections.singletonList(agentIp), null, null);
if (certificate == null || certificate.getClientCertificate() == null) {
throw new CloudRuntimeException("Failed to issue certificates for KVM host agent: " + agentIp);
}
final SetupCertificateCommand certificateCommand = new SetupCertificateCommand(certificate);
final SSHCmdHelper.SSHCmdResult setupCertResult = SSHCmdHelper.sshExecuteCmdWithResult(sshConnection,
String.format("sudo /usr/share/cloudstack-common/scripts/util/%s " +
"/etc/cloudstack/agent/agent.properties %s " +
"/etc/cloudstack/agent/%s %s " +
"/etc/cloudstack/agent/%s \"%s\" " +
"/etc/cloudstack/agent/%s \"%s\" " +
"/etc/cloudstack/agent/%s \"%s\"",
KeyStoreUtils.KS_IMPORT_SCRIPT,
keystorePassword,
KeyStoreUtils.KS_FILENAME,
KeyStoreUtils.SSH_MODE,
KeyStoreUtils.CERT_FILENAME,
certificateCommand.getEncodedCertificate(),
KeyStoreUtils.CACERT_FILENAME,
certificateCommand.getEncodedCaCertificates(),
KeyStoreUtils.PKEY_FILENAME,
certificateCommand.getEncodedPrivateKey()));
if (setupCertResult != null && !setupCertResult.isSuccess()) {
throw new CloudRuntimeException("Failed to setup certificate in the KVM agent's keystore file, please see logs and configure manually!");
}
caManager.provisionCertificateViaSsh(sshConnection, agentIp, agentHostname);
if (logger.isDebugEnabled()) {
logger.debug("Succeeded to import certificate in the keystore for agent on the KVM host: " + agentIp + ". Agent secured and trusted.");

167
server/src/main/java/org/apache/cloudstack/ca/CAManagerImpl.java
View File

@ -29,6 +29,7 @@ import java.security.cert.CertificateParsingException;
import java.security.cert.X509Certificate;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
@ -43,6 +44,18 @@ import javax.net.ssl.SSLEngine;
import javax.net.ssl.TrustManagerFactory;
import javax.net.ssl.X509TrustManager;
import com.trilead.ssh2.Connection;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import com.cloud.host.HostVO;
import com.cloud.utils.PasswordGenerator;
import com.cloud.utils.ssh.SSHCmdHelper;
import com.cloud.vm.VMInstanceVO;
import com.cloud.vm.dao.VMInstanceDao;
import org.apache.cloudstack.utils.security.KeyStoreUtils;
import org.apache.commons.lang3.math.NumberUtils;
import org.apache.cloudstack.api.ApiErrorCode;
import org.apache.cloudstack.api.ServerApiException;
import org.apache.cloudstack.api.command.admin.ca.IssueCertificateCmd;
@ -63,6 +76,7 @@ import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import com.cloud.agent.AgentManager;
import com.cloud.agent.api.routing.NetworkElementCommand;
import com.cloud.alert.AlertManager;
import com.cloud.certificate.CrlVO;
import com.cloud.certificate.dao.CrlDao;
@ -84,6 +98,12 @@ public class CAManagerImpl extends ManagerBase implements CAManager {
@Inject
private HostDao hostDao;
@Inject
private VMInstanceDao vmInstanceDao;
@Inject
private NetworkOrchestrationService networkOrchestrationService;
@Inject
private ConfigurationDao configDao;
@Inject
private AgentManager agentManager;
@Inject
private BackgroundPollManager backgroundPollManager;
@ -180,12 +200,17 @@ public class CAManagerImpl extends ManagerBase implements CAManager {
@Override
@ActionEvent(eventType = EventTypes.EVENT_CA_CERTIFICATE_PROVISION, eventDescription = "provisioning certificate for host", async = true)
public boolean provisionCertificate(final Host host, final Boolean reconnect, final String caProvider) {
public boolean provisionCertificate(final Host host, final Boolean reconnect, final String caProvider, final boolean forced) {
if (host == null) {
throw new CloudRuntimeException("Unable to find valid host to renew certificate for");
}
CallContext.current().setEventDetails("Host ID: " + host.getUuid());
CallContext.current().putContextParameter(Host.class, host.getUuid());
if (forced) {
return provisionCertificateForced(host, reconnect, caProvider);
}
String csr = null;
try {
@ -203,6 +228,138 @@ public class CAManagerImpl extends ManagerBase implements CAManager {
}
}
private boolean provisionCertificateForced(Host host, Boolean reconnect, String caProvider) {
if (host.getType() == Host.Type.Routing && host.getHypervisorType() == com.cloud.hypervisor.Hypervisor.HypervisorType.KVM) {
return provisionKvmHostViaSsh(host);
} else if (host.getType() == Host.Type.ConsoleProxy || host.getType() == Host.Type.SecondaryStorageVM) {
return provisionSystemVmViaSsh(host, reconnect);
}
throw new CloudRuntimeException("Forced certificate provisioning is only supported for KVM hosts and SystemVMs.");
}
@Override
public void provisionCertificateViaSsh(final Connection sshConnection, final String agentIp, final String agentHostname) {
Integer validityPeriod = CAManager.CertValidityPeriod.value();
if (validityPeriod < 1) {
validityPeriod = 1;
}
String keystorePassword = PasswordGenerator.generateRandomPassword(16);
// 1. Setup Keystore and Generate CSR
final SSHCmdHelper.SSHCmdResult keystoreSetupResult = SSHCmdHelper.sshExecuteCmdWithResult(sshConnection,
String.format("sudo /usr/share/cloudstack-common/scripts/util/%s " +
"/etc/cloudstack/agent/agent.properties " +
"/etc/cloudstack/agent/%s " +
"%s %d " +
"/etc/cloudstack/agent/%s",
KeyStoreUtils.KS_SETUP_SCRIPT,
KeyStoreUtils.KS_FILENAME,
keystorePassword,
validityPeriod,
KeyStoreUtils.CSR_FILENAME));
if (!keystoreSetupResult.isSuccess()) {
throw new CloudRuntimeException("Failed to setup keystore and generate CSR via SSH on host: " + agentIp);
}
// 2. Issue Certificate based on returned CSR
final String csr = keystoreSetupResult.getStdOut();
final Certificate certificate = issueCertificate(csr, Arrays.asList(agentHostname, agentIp),
Collections.singletonList(agentIp), null, null);
if (certificate == null || certificate.getClientCertificate() == null) {
throw new CloudRuntimeException("Failed to issue certificates for host: " + agentIp);
}
// 3. Import Certificate into agent keystore
final SetupCertificateCommand certificateCommand = new SetupCertificateCommand(certificate);
final SSHCmdHelper.SSHCmdResult setupCertResult = SSHCmdHelper.sshExecuteCmdWithResult(sshConnection,
String.format("sudo /usr/share/cloudstack-common/scripts/util/%s " +
"/etc/cloudstack/agent/agent.properties %s " +
"/etc/cloudstack/agent/%s %s " +
"/etc/cloudstack/agent/%s \"%s\" " +
"/etc/cloudstack/agent/%s \"%s\" " +
"/etc/cloudstack/agent/%s \"%s\"",
KeyStoreUtils.KS_IMPORT_SCRIPT,
keystorePassword,
KeyStoreUtils.KS_FILENAME,
KeyStoreUtils.SSH_MODE,
KeyStoreUtils.CERT_FILENAME,
certificateCommand.getEncodedCertificate(),
KeyStoreUtils.CACERT_FILENAME,
certificateCommand.getEncodedCaCertificates(),
KeyStoreUtils.PKEY_FILENAME,
certificateCommand.getEncodedPrivateKey()));
if (!setupCertResult.isSuccess()) {
throw new CloudRuntimeException("Failed to import certificates into agent keystore via SSH on host: " + agentIp);
}
}
private boolean provisionKvmHostViaSsh(Host host) {
final HostVO hostVO = (HostVO) host;
hostDao.loadDetails(hostVO);
String username = hostVO.getDetail(ApiConstants.USERNAME);
String password = hostVO.getDetail(ApiConstants.PASSWORD);
String hostIp = host.getPrivateIpAddress();
int port = AgentManager.KVMHostDiscoverySshPort.valueIn(host.getClusterId());
if (hostVO.getDetail(Host.HOST_SSH_PORT) != null) {
port = NumberUtils.toInt(hostVO.getDetail(Host.HOST_SSH_PORT), port);
}
Connection sshConnection = null;
try {
sshConnection = new Connection(hostIp, port);
sshConnection.connect(null, 60000, 60000);
String privateKey = configDao.getValue("ssh.privatekey");
if (!SSHCmdHelper.acquireAuthorizedConnectionWithPublicKey(sshConnection, username, privateKey)) {
if (StringUtils.isEmpty(password) || !sshConnection.authenticateWithPassword(username, password)) {
throw new CloudRuntimeException("Failed to authenticate to host via SSH for forced provisioning: " + hostIp);
}
}
provisionCertificateViaSsh(sshConnection, hostIp, host.getName());
SSHCmdHelper.sshExecuteCmd(sshConnection, "sudo service cloudstack-agent restart");
return true;
} catch (Exception e) {
logger.error("Error during forced SSH provisioning for KVM host " + host.getUuid(), e);
return false;
} finally {
if (sshConnection != null) {
sshConnection.close();
}
}
}
private boolean provisionSystemVmViaSsh(Host host, Boolean reconnect) {
VMInstanceVO vm = vmInstanceDao.findVMByInstanceName(host.getName());
if (vm == null) {
throw new CloudRuntimeException("Cannot find underlying VM for host: " + host.getName());
}
final Map<String, String> sshAccessDetails = networkOrchestrationService.getSystemVMAccessDetails(vm);
final Map<String, String> ipAddressDetails = new HashMap<>(sshAccessDetails);
ipAddressDetails.remove(NetworkElementCommand.ROUTER_NAME);
try {
final Host hypervisorHost = hostDao.findById(vm.getHostId());
if (hypervisorHost == null) {
throw new CloudRuntimeException("Cannot find hypervisor host for system VM: " + host.getName());
}
final Certificate certificate = issueCertificate(null, Arrays.asList(vm.getHostName(), vm.getInstanceName()),
new ArrayList<>(ipAddressDetails.values()), CertValidityPeriod.value(), null);
return deployCertificate(hypervisorHost, certificate, reconnect, sshAccessDetails);
} catch (Exception e) {
logger.error("Failed to provision system VM " + host.getName() + " via hypervisor SSH proxy. Ensure the hypervisor host is connected.", e);
return false;
}
}
@Override
public String generateKeyStoreAndCsr(final Host host, final Map<String, String> sshAccessDetails) throws AgentUnavailableException, OperationTimedoutException {
final SetupKeyStoreCommand cmd = new SetupKeyStoreCommand(CertValidityPeriod.value());
@ -343,7 +500,7 @@ public class CAManagerImpl extends ManagerBase implements CAManager {
if (AutomaticCertRenewal.valueIn(host.getClusterId())) {
try {
logger.debug("Attempting certificate auto-renewal for " + hostDescription, e);
boolean result = caManager.provisionCertificate(host, false, null);
boolean result = caManager.provisionCertificate(host, false, null, false);
if (result) {
logger.debug("Succeeded in auto-renewing certificate for " + hostDescription, e);
} else {
@ -421,12 +578,12 @@ public class CAManagerImpl extends ManagerBase implements CAManager {
trustStore.load(null, null);
// Copy existing default trusted certs
final TrustManagerFactory defaultTmf = TrustManagerFactory.getInstance("SunX509");
final TrustManagerFactory defaultTmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
defaultTmf.init((KeyStore) null);
final X509TrustManager defaultTm = (X509TrustManager) defaultTmf.getTrustManagers()[0];
int aliasIndex = 0;
for (final X509Certificate cert : defaultTm.getAcceptedIssuers()) {
final String alias = cert.getSubjectX500Principal().getName();
trustStore.setCertificateEntry(alias, cert);
trustStore.setCertificateEntry("default-ca-" + aliasIndex++, cert);
}
// Add CA provider's certificates

10
server/src/test/java/org/apache/cloudstack/ca/CABackgroundTaskTest.java
View File

@ -115,19 +115,19 @@ public class CABackgroundTaskTest {
certMap.put(hostIp, expiredCertificate);
Assume.assumeThat(certMap.size() == 1, is(true));
task.runInContext();
Mockito.verify(caManager, Mockito.times(1)).provisionCertificate(host, false, null);
Mockito.verify(caManager, Mockito.times(1)).provisionCertificate(host, false, null, false);
Mockito.verify(caManager, Mockito.times(0)).sendAlert(Mockito.any(Host.class), Mockito.anyString(), Mockito.anyString());
}
@Test
public void testAutoRenewalEnabledWithExceptionsOnProvisioning() throws Exception {
overrideDefaultConfigValue(AutomaticCertRenewal, "_defaultValue", "true");
Mockito.when(caManager.provisionCertificate(any(Host.class), anyBoolean(), nullable(String.class))).thenThrow(new CloudRuntimeException("some error"));
Mockito.when(caManager.provisionCertificate(any(Host.class), anyBoolean(), nullable(String.class), anyBoolean())).thenThrow(new CloudRuntimeException("some error"));
host.setManagementServerId(ManagementServerNode.getManagementServerId());
certMap.put(hostIp, expiredCertificate);
Assume.assumeThat(certMap.size() == 1, is(true));
task.runInContext();
Mockito.verify(caManager, Mockito.times(1)).provisionCertificate(host, false, null);
Mockito.verify(caManager, Mockito.times(1)).provisionCertificate(host, false, null, false);
Mockito.verify(caManager, Mockito.times(1)).sendAlert(Mockito.any(Host.class), Mockito.anyString(), Mockito.anyString());
}
@ -138,12 +138,12 @@ public class CABackgroundTaskTest {
Assume.assumeThat(certMap.size() == 1, is(true));
// First round
task.runInContext();
Mockito.verify(caManager, Mockito.times(0)).provisionCertificate(Mockito.any(Host.class), anyBoolean(), Mockito.anyString());
Mockito.verify(caManager, Mockito.times(0)).provisionCertificate(Mockito.any(Host.class), anyBoolean(), Mockito.anyString(), Mockito.anyBoolean());
Mockito.verify(caManager, Mockito.times(1)).sendAlert(Mockito.any(Host.class), Mockito.anyString(), Mockito.anyString());
Mockito.reset(caManager);
// Second round
task.runInContext();
Mockito.verify(caManager, Mockito.times(0)).provisionCertificate(Mockito.any(Host.class), anyBoolean(), Mockito.anyString());
Mockito.verify(caManager, Mockito.times(0)).provisionCertificate(Mockito.any(Host.class), anyBoolean(), Mockito.anyString(), Mockito.anyBoolean());
Mockito.verify(caManager, Mockito.times(0)).sendAlert(Mockito.any(Host.class), Mockito.anyString(), Mockito.anyString());
}

2
server/src/test/java/org/apache/cloudstack/ca/CAManagerImplTest.java
View File

@ -121,7 +121,7 @@ public class CAManagerImplTest {
Mockito.when(agentManager.send(anyLong(), any(SetupCertificateCommand.class))).thenReturn(new SetupCertificateAnswer(true));
Mockito.when(agentManager.send(anyLong(), any(SetupKeyStoreCommand.class))).thenReturn(new SetupKeystoreAnswer("someCsr"));
Mockito.doNothing().when(agentManager).reconnect(Mockito.anyLong());
Assert.assertTrue(caManager.provisionCertificate(host, true, null));
Assert.assertTrue(caManager.provisionCertificate(host, true, null, false));
Mockito.verify(agentManager, Mockito.times(1)).send(Mockito.anyLong(), any(SetupKeyStoreCommand.class));
Mockito.verify(agentManager, Mockito.times(1)).send(Mockito.anyLong(), any(SetupCertificateCommand.class));
Mockito.verify(agentManager, Mockito.times(1)).reconnect(Mockito.anyLong());

2
systemvm/patch-sysvms.sh
View File

@ -139,7 +139,7 @@ patch_systemvm() {
CACERT_FILE="/usr/local/share/ca-certificates/cloudstack/ca.crt"
if [ -f "$CACERT_FILE" ] && [ -f "$REALHOSTIP_KS_FILE" ]; then
awk '/-----BEGIN CERTIFICATE-----/{n++}{print > "cloudca." n }' "$CACERT_FILE"
awk 'BEGIN{n=0} /-----BEGIN CERTIFICATE-----/{n++}{print > "cloudca." n }' "$CACERT_FILE"
for caChain in $(ls cloudca.* 2>/dev/null); do
keytool -delete -noprompt -alias "$caChain" -keystore "$REALHOSTIP_KS_FILE" \
-storepass "$REALHOSTIP_PASS" > /dev/null 2>&1 || true