Add root CA to the truststore for System VMs

api/src/main/java/org/apache/cloudstack/ca/CAManager.java

@@ -39,7 +39,10 @@ public interface CAManager extends CAService, Configurable, PluggableService {
     ConfigKey<String> CAProviderPlugin = new ConfigKey<>("Advanced", String.class,
             "ca.framework.provider.plugin",
             "root",
-            "The CA provider plugin that is used for secure CloudStack management server-agent communication for encryption and authentication. Restart management server(s) when changed.", true);
+            "The CA provider plugin used for CloudStack internal certificate management (MS-agent encryption and authentication). " +
+            "The default 'root' provider auto-generates a CA on first startup, but also supports user-provided custom CA material " +
+            "via the ca.plugin.root.private.key, ca.plugin.root.public.key, and ca.plugin.root.ca.certificate settings. " +
+            "Restart management server(s) when changed.", true);
 
     ConfigKey<Integer> CertKeySize = new ConfigKey<>("Advanced", Integer.class,
                                     "ca.framework.cert.keysize",

plugins/ca/root-ca/src/main/java/org/apache/cloudstack/ca/provider/RootCAProvider.java

@@ -106,17 +106,21 @@ public final class RootCAProvider extends AdapterBase implements CAProvider, Con
     private static ConfigKey<String> rootCAPrivateKey = new ConfigKey<>("Hidden", String.class,
             "ca.plugin.root.private.key",
             null,
-            "The ROOT CA private key.", true);
+            "The ROOT CA private key in PEM format (PKCS#8: must start with '-----BEGIN PRIVATE KEY-----'). " +
+            "When set along with the public key and certificate, CloudStack uses this custom CA instead of auto-generating one. " +
+            "All three ca.plugin.root.* keys must be set together. Restart management server(s) when changed.", true);
 
     private static ConfigKey<String> rootCAPublicKey = new ConfigKey<>("Hidden", String.class,
             "ca.plugin.root.public.key",
             null,
-            "The ROOT CA public key.", true);
+            "The ROOT CA public key in PEM format (X.509/SPKI: must start with '-----BEGIN PUBLIC KEY-----'). " +
+            "Required when providing a custom CA. Restart management server(s) when changed.", true);
 
     private static ConfigKey<String> rootCACertificate = new ConfigKey<>("Hidden", String.class,
             "ca.plugin.root.ca.certificate",
             null,
-            "The ROOT CA certificate.", true);
+            "The ROOT CA X.509 certificate in PEM format (must start with '-----BEGIN CERTIFICATE-----'). " +
+            "Required when providing a custom CA. Restart management server(s) when changed.", true);
 
     private static ConfigKey<String> rootCAIssuerDN = new ConfigKey<>("Advanced", String.class,
             "ca.plugin.root.issuer.dn",
@@ -422,13 +426,29 @@ public final class RootCAProvider extends AdapterBase implements CAProvider, Con
 
 
     private boolean setupCA() {
-        if (!loadRootCAKeyPair() && !saveNewRootCAKeypair()) {
-            logger.error("Failed to save and load root CA keypair");
-            return false;
+        if (!loadRootCAKeyPair()) {
+            if (hasUserProvidedCAKeys()) {
+                logger.error("Failed to load user-provided CA keys from configuration. " +
+                    "Check that ca.plugin.root.private.key, ca.plugin.root.public.key, and " +
+                    "ca.plugin.root.ca.certificate are all set and in the correct PEM format " +
+                    "(private key must be PKCS#8: '-----BEGIN PRIVATE KEY-----'). " +
+                    "Overwriting with auto-generated keys.");
+            }
+            if (!saveNewRootCAKeypair()) {
+                logger.error("Failed to save and load root CA keypair");
+                return false;
+            }
         }
-        if (!loadRootCACertificate() && !saveNewRootCACertificate()) {
-            logger.error("Failed to save and load root CA certificate");
-            return false;
+        if (!loadRootCACertificate()) {
+            if (hasUserProvidedCAKeys()) {
+                logger.error("Failed to load user-provided CA certificate. " +
+                    "Check that ca.plugin.root.ca.certificate is set and in PEM format. " +
+                    "Overwriting with auto-generated certificate.");
+            }
+            if (!saveNewRootCACertificate()) {
+                logger.error("Failed to save and load root CA certificate");
+                return false;
+            }
         }
         if (!loadManagementKeyStore()) {
             logger.error("Failed to check and configure management server keystore");
@@ -437,10 +457,16 @@ public final class RootCAProvider extends AdapterBase implements CAProvider, Con
         return true;
     }
 
+    private boolean hasUserProvidedCAKeys() {
+        return StringUtils.isNotEmpty(rootCAPublicKey.value())
+            || StringUtils.isNotEmpty(rootCAPrivateKey.value())
+            || StringUtils.isNotEmpty(rootCACertificate.value());
+    }
+
     @Override
     public boolean start() {
         managementCertificateCustomSAN = CAManager.CertManagementCustomSubjectAlternativeName.value();
-        return loadRootCAKeyPair() && loadRootCAKeyPair() && loadManagementKeyStore();
+        return loadRootCAKeyPair() && loadRootCACertificate() && loadManagementKeyStore();
     }
 
     @Override

scripts/util/keystore-cert-import

@@ -137,6 +137,22 @@ if [ -f "$SYSTEM_FILE" ]; then
     chmod 644 /usr/local/share/ca-certificates/cloudstack/ca.crt
     update-ca-certificates > /dev/null 2>&1 || true
 
+    # Import CA cert(s) into realhostip.keystore so the SSVM JVM
+    # (which overrides the truststore via -Djavax.net.ssl.trustStore in _run.sh)
+    # can trust servers signed by the CloudStack CA
+    REALHOSTIP_KS_FILE="$(dirname $(dirname $PROPS_FILE))/certs/realhostip.keystore"
+    REALHOSTIP_PASS="vmops.com"
+    if [ -f "$REALHOSTIP_KS_FILE" ]; then
+        awk '/-----BEGIN CERTIFICATE-----/{n++}{print > "cloudca." n }' "$CACERT_FILE"
+        for caChain in $(ls cloudca.* 2>/dev/null); do
+            keytool -delete -noprompt -alias "$caChain" -keystore "$REALHOSTIP_KS_FILE" \
+                -storepass "$REALHOSTIP_PASS" > /dev/null 2>&1 || true
+            keytool -import -noprompt -trustcacerts -alias "$caChain" -file "$caChain" \
+                -keystore "$REALHOSTIP_KS_FILE" -storepass "$REALHOSTIP_PASS" > /dev/null 2>&1
+        done
+        rm -f cloudca.*
+    fi
+
     # Ensure cloud service is running in systemvm
     if [ "$MODE" == "ssh" ]; then
         systemctl start cloud > /dev/null 2>&1

systemvm/patch-sysvms.sh

@@ -126,7 +126,28 @@ patch_systemvm() {
 
   if [ "$TYPE" = "consoleproxy" ] || [ "$TYPE" = "secstorage" ]; then
     # Import global cacerts into 'cloud' service's keystore
-    keytool -importkeystore -srckeystore /etc/ssl/certs/java/cacerts -destkeystore /usr/local/cloud/systemvm/certs/realhostip.keystore -srcstorepass changeit -deststorepass vmops.com -noprompt 2>/dev/null || true
+    REALHOSTIP_KS_FILE="/usr/local/cloud/systemvm/certs/realhostip.keystore"
+    REALHOSTIP_PASS="vmops.com"
+
+    keytool -importkeystore -srckeystore /etc/ssl/certs/java/cacerts \
+        -destkeystore "$REALHOSTIP_KS_FILE" -srcstorepass changeit -deststorepass \
+        "$REALHOSTIP_PASS" -noprompt 2>/dev/null || true
+
+    # Import CA cert(s) into realhostip.keystore so the SSVM JVM
+    # (which overrides the truststore via -Djavax.net.ssl.trustStore in _run.sh)
+    # can trust servers signed by the CloudStack CA
+    CACERT_FILE="/usr/local/share/ca-certificates/cloudstack/ca.crt"
+
+    if [ -f "$CACERT_FILE" ] && [ -f "$REALHOSTIP_KS_FILE" ]; then
+        awk '/-----BEGIN CERTIFICATE-----/{n++}{print > "cloudca." n }' "$CACERT_FILE"
+        for caChain in $(ls cloudca.* 2>/dev/null); do
+            keytool -delete -noprompt -alias "$caChain" -keystore "$REALHOSTIP_KS_FILE" \
+                -storepass "$REALHOSTIP_PASS" > /dev/null 2>&1 || true
+            keytool -import -noprompt -trustcacerts -alias "$caChain" -file "$caChain" \
+                -keystore "$REALHOSTIP_KS_FILE" -storepass "$REALHOSTIP_PASS" > /dev/null 2>&1
+        done
+        rm -f cloudca.*
+    fi
   fi
 
   update_checksum $newpath/cloud-scripts.tgz

utils/src/main/java/com/cloud/utils/nio/Link.java

@@ -552,7 +552,7 @@ public class Link {
             LOGGER.error(String.format("SSL error caught during wrap data: %s, for local address=%s, remote address=%s.",
                     sslException.getMessage(), socketChannel.getLocalAddress(), socketChannel.getRemoteAddress()));
             sslEngine.closeOutbound();
-            return new HandshakeHolder(myAppData, myNetData, true);
+            return new HandshakeHolder(myAppData, myNetData, false);
         }
         if (result == null) {
             return new HandshakeHolder(myAppData, myNetData, false);

utils/src/main/java/org/apache/cloudstack/utils/security/KeyStoreUtils.java

@@ -26,7 +26,6 @@ import com.cloud.utils.PropertiesUtil;
 public class KeyStoreUtils {
     public static final String KS_SETUP_SCRIPT = "keystore-setup";
     public static final String KS_IMPORT_SCRIPT = "keystore-cert-import";
-    public static final String KS_SYSTEMVM_IMPORT_SCRIPT = "keystore-cert-import-sysvm";
 
     public static final String AGENT_PROPSFILE = "agent.properties";
     public static final String KS_PASSPHRASE_PROPERTY = "keystore.passphrase";