From 0a0ea4c642783eeba51d86e80e4cf567a4537488 Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Mon, 5 Mar 2018 15:26:31 +0100 Subject: [PATCH] APPLE-FR29: Secure KVM Live VM Migration (FRO-93) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This extends securing of KVM hosts to securing of libvirt on KVM host as well for TLS enabled live VM migration. Based on whether keystore and certificates files are available at /etc/cloudstack/agent, the KVM agent determines whether to use TLS or TCP based uris for live VM migration. It is also enforced that a secured host will allow live VM migration to/from other secured host, and an unsecured hosts will allow live VM migration to/from other unsecured host only. Post upgrade the KVM agent on startup will expose its security state (secured detail is sent as true or false) to the managements server that gets saved in host_details for the host. This host detail can be accesed via the listHosts response, and in the UI unsecured KVM hosts will show up with the host state of ‘unsecured’. Further, a button has been added that allows admins to provision/renew certificates to KVM hosts and can be used to secure any unsecured KVM host. The `cloudstack-setup-agent` was modified to accept a new flag ‘-s’ which reconfigured libvirtd with following settings that enables only TLS: listen_tcp=0 listen_tls=1 tcp_port="16509" auth_tcp="none" tls_port=”16514” auth_tls=”none” key_file = "/etc/pki/libvirt/private/serverkey.pem" cert_file = "/etc/pki/libvirt/servercert.pem" ca_file = "/etc/pki/CA/cacert.pem" For a connected KVM host agent, when the certificate are renewed/provisioned a background task is scheduled that waits until all of the agent tasks finish after which libvirt process is restarted and finally the agent is restarted via AgentShell. There are no API or DB changes. Signed-off-by: Rohit Yadav --- agent/bindir/cloud-setup-agent.in | 9 ++ agent/src/com/cloud/agent/Agent.java | 146 ++++++++++++++++-- agent/src/com/cloud/agent/AgentShell.java | 7 +- agent/src/com/cloud/agent/IAgentShell.java | 5 + .../classes/resources/messages.properties | 2 + .../VirtualRoutingResource.java | 18 +-- .../ca/PostCertificateRenewalCommand.java | 34 ++++ .../ca/SetupCertificateCommand.java | 6 +- debian/cloudstack-agent.postinst | 8 + packaging/centos63/cloud.spec | 8 + .../ca/provider/RootCAProvider.java | 8 +- .../resource/LibvirtComputingResource.java | 64 ++++++-- .../LibvirtComputingResourceTest.java | 23 ++- python/lib/cloud_utils.py | 2 +- python/lib/cloudutils/serviceConfig.py | 52 ++++--- scripts/util/keystore-cert-import | 12 ++ scripts/util/keystore-setup | 4 +- .../discoverer/LibvirtServerDiscoverer.java | 34 ++-- .../apache/cloudstack/ca/CAManagerImpl.java | 3 +- ui/css/cloudstack3.css | 2 + ui/dictionary.jsp | 1 + ui/dictionary2.jsp | 1 + ui/scripts/system.js | 57 ++++++- utils/src/com/cloud/utils/nio/Link.java | 8 +- utils/src/com/cloud/utils/script/Script.java | 2 +- .../src/com/cloud/utils/ssh/SSHCmdHelper.java | 4 +- .../cloudstack/utils/security/CertUtils.java | 11 +- .../utils/security/KeyStoreUtils.java | 35 +++-- 28 files changed, 439 insertions(+), 127 deletions(-) create mode 100644 core/src/org/apache/cloudstack/ca/PostCertificateRenewalCommand.java diff --git a/agent/bindir/cloud-setup-agent.in b/agent/bindir/cloud-setup-agent.in index e8f8f2f452f..0087a974c04 100755 --- a/agent/bindir/cloud-setup-agent.in +++ b/agent/bindir/cloud-setup-agent.in @@ -26,6 +26,7 @@ from cloudutils.configFileOps import configFileOps from cloudutils.globalEnv import globalEnv from cloudutils.networkConfig import networkConfig from cloudutils.syscfg import sysConfigFactory +from cloudutils.serviceConfig import configureLibvirtConfig from optparse import OptionParser @@ -100,6 +101,7 @@ if __name__ == '__main__': parser.add_option("-c", "--cluster", dest="cluster", help="cluster id") parser.add_option("-t", "--hypervisor", default="kvm", dest="hypervisor", help="hypervisor type") parser.add_option("-g", "--guid", dest="guid", help="guid") + parser.add_option("-s", action="store_true", default=False, dest="secure", help="Secure and enable TLS for libvirtd") parser.add_option("--pubNic", dest="pubNic", help="Public traffic interface") parser.add_option("--prvNic", dest="prvNic", help="Private traffic interface") parser.add_option("--guestNic", dest="guestNic", help="Guest traffic interface") @@ -110,6 +112,12 @@ if __name__ == '__main__': glbEnv.bridgeType = bridgeType (options, args) = parser.parse_args() + + if not options.auto and options.secure: + configureLibvirtConfig(True) + print "Libvirtd with TLS configured" + sys.exit(0) + if options.auto is None: userInputs = getUserInputs() glbEnv.mgtSvr = userInputs[0] @@ -138,6 +146,7 @@ if __name__ == '__main__': glbEnv.nics.append(options.prvNic) glbEnv.nics.append(options.pubNic) glbEnv.nics.append(options.guestNic) + glbEnv.secure = options.secure print "Starting to configure your system:" syscfg = sysConfigFactory.getSysConfigFactory(glbEnv) diff --git a/agent/src/com/cloud/agent/Agent.java b/agent/src/com/cloud/agent/Agent.java index 42543221624..6c2d42afc90 100755 --- a/agent/src/com/cloud/agent/Agent.java +++ b/agent/src/com/cloud/agent/Agent.java @@ -41,6 +41,7 @@ import javax.naming.ConfigurationException; import org.apache.cloudstack.agent.lb.SetupMSListAnswer; import org.apache.cloudstack.agent.lb.SetupMSListCommand; +import org.apache.cloudstack.ca.PostCertificateRenewalCommand; import org.apache.cloudstack.ca.SetupCertificateAnswer; import org.apache.cloudstack.ca.SetupCertificateCommand; import org.apache.cloudstack.ca.SetupKeyStoreCommand; @@ -67,6 +68,7 @@ import com.cloud.agent.api.StartupCommand; import com.cloud.agent.transport.Request; import com.cloud.agent.transport.Response; import com.cloud.exception.AgentControlChannelException; +import com.cloud.host.Host; import com.cloud.resource.ServerResource; import com.cloud.utils.PropertiesUtil; import com.cloud.utils.StringUtils; @@ -126,6 +128,7 @@ public class Agent implements HandlerFactory, IAgentControl { Long _id; Timer _timer = new Timer("Agent Timer"); + Timer certTimer; Timer hostLBTimer; List _watchList = new ArrayList(); @@ -139,9 +142,11 @@ public class Agent implements HandlerFactory, IAgentControl { long _startupWait = _startupWaitDefault; boolean _reconnectAllowed = true; //For time sentitive task, e.g. PingTask - private final ThreadPoolExecutor _ugentTaskPool; + ThreadPoolExecutor _ugentTaskPool; ExecutorService _executor; + Thread _shutdownThread = new ShutdownThread(this); + private String _keystoreSetupPath; private String _keystoreCertImportPath; @@ -152,7 +157,7 @@ public class Agent implements HandlerFactory, IAgentControl { _connection = new NioClient("Agent", _shell.getNextHost(), _shell.getPort(), _shell.getWorkers(), this); - Runtime.getRuntime().addShutdownHook(new ShutdownThread(this)); + Runtime.getRuntime().addShutdownHook(_shutdownThread); _ugentTaskPool = new ThreadPoolExecutor(shell.getPingRetries(), 2 * shell.getPingRetries(), 10, TimeUnit.MINUTES, new SynchronousQueue(), new NamedThreadFactory( @@ -191,7 +196,7 @@ public class Agent implements HandlerFactory, IAgentControl { // ((NioClient)_connection).setBindAddress(_shell.getPrivateIp()); s_logger.debug("Adding shutdown hook"); - Runtime.getRuntime().addShutdownHook(new ShutdownThread(this)); + Runtime.getRuntime().addShutdownHook(_shutdownThread); _ugentTaskPool = new ThreadPoolExecutor(shell.getPingRetries(), 2 * shell.getPingRetries(), 10, TimeUnit.MINUTES, new SynchronousQueue(), new NamedThreadFactory( @@ -244,14 +249,14 @@ public class Agent implements HandlerFactory, IAgentControl { throw new CloudRuntimeException("Unable to start the resource: " + _resource.getName()); } - _keystoreSetupPath = Script.findScript("scripts/util/", KeyStoreUtils.keyStoreSetupScript); + _keystoreSetupPath = Script.findScript("scripts/util/", KeyStoreUtils.KS_SETUP_SCRIPT); if (_keystoreSetupPath == null) { - throw new CloudRuntimeException(String.format("Unable to find the '%s' script", KeyStoreUtils.keyStoreSetupScript)); + throw new CloudRuntimeException(String.format("Unable to find the '%s' script", KeyStoreUtils.KS_SETUP_SCRIPT)); } - _keystoreCertImportPath = Script.findScript("scripts/util/", KeyStoreUtils.keyStoreImportScript); + _keystoreCertImportPath = Script.findScript("scripts/util/", KeyStoreUtils.KS_IMPORT_SCRIPT); if (_keystoreCertImportPath == null) { - throw new CloudRuntimeException(String.format("Unable to find the '%s' script", KeyStoreUtils.keyStoreImportScript)); + throw new CloudRuntimeException(String.format("Unable to find the '%s' script", KeyStoreUtils.KS_IMPORT_SCRIPT)); } try { @@ -273,6 +278,19 @@ public class Agent implements HandlerFactory, IAgentControl { } } _shell.updateConnectedHost(); + + // In case of software based restart, GC to remove old instances + _executor.submit(new Runnable() { + @Override + public void run() { + try { + Thread.sleep(2000L); + } catch (final InterruptedException ignored) { + } finally { + System.gc(); + } + } + }); } public void stop(final String reason, final String detail) { @@ -297,6 +315,7 @@ public class Agent implements HandlerFactory, IAgentControl { } _connection.stop(); _connection = null; + _link = null; } if (_resource != null) { @@ -304,7 +323,34 @@ public class Agent implements HandlerFactory, IAgentControl { _resource = null; } - _ugentTaskPool.shutdownNow(); + if (_startup != null) { + _startup = null; + } + + if (_ugentTaskPool != null) { + _ugentTaskPool.shutdownNow(); + _ugentTaskPool = null; + } + + if (_executor != null) { + _executor.shutdown(); + _executor = null; + } + + if (_timer != null) { + _timer.cancel(); + _timer = null; + } + + if (hostLBTimer != null) { + hostLBTimer.cancel(); + hostLBTimer = null; + } + + if (certTimer != null) { + certTimer.cancel(); + certTimer = null; + } } public Long getId() { @@ -317,6 +363,15 @@ public class Agent implements HandlerFactory, IAgentControl { _shell.setPersistentProperty(getResourceName(), "id", Long.toString(id)); } + private synchronized void scheduleServicesRestartTask() { + if (certTimer != null) { + certTimer.cancel(); + certTimer.purge(); + } + certTimer = new Timer("Certificate Renewal Timer"); + certTimer.schedule(new PostCertificateRenewalTask(this), 5000L); + } + private synchronized void scheduleHostLBCheckerTask(final long checkInterval) { if (hostLBTimer != null) { hostLBTimer.cancel(); @@ -577,6 +632,9 @@ public class Agent implements HandlerFactory, IAgentControl { answer = setupAgentKeystore((SetupKeyStoreCommand) cmd); } else if (cmd instanceof SetupCertificateCommand && ((SetupCertificateCommand) cmd).isHandleByAgent()) { answer = setupAgentCertificate((SetupCertificateCommand) cmd); + if (Host.Type.Routing.equals(_resource.getType())) { + scheduleServicesRestartTask(); + } } else if (cmd instanceof SetupMSListCommand) { answer = setupManagementServerList((SetupMSListCommand) cmd); } else { @@ -638,13 +696,13 @@ public class Agent implements HandlerFactory, IAgentControl { if (agentFile == null) { return new Answer(cmd, false, "Failed to find agent.properties file"); } - final String keyStoreFile = agentFile.getParent() + KeyStoreUtils.defaultKeystoreFile; - final String csrFile = agentFile.getParent() + KeyStoreUtils.defaultCsrFile; + final String keyStoreFile = agentFile.getParent() + KeyStoreUtils.KS_FILENAME; + final String csrFile = agentFile.getParent() + KeyStoreUtils.CSR_FILENAME; - String storedPassword = _shell.getPersistentProperty(null, KeyStoreUtils.passphrasePropertyName); + String storedPassword = _shell.getPersistentProperty(null, KeyStoreUtils.KS_PASSPHRASE_PROPERTY); if (Strings.isNullOrEmpty(storedPassword)) { storedPassword = keyStorePassword; - _shell.setPersistentProperty(null, KeyStoreUtils.passphrasePropertyName, storedPassword); + _shell.setPersistentProperty(null, KeyStoreUtils.KS_PASSPHRASE_PROPERTY, storedPassword); } Script script = new Script(true, _keystoreSetupPath, 60000, s_logger); @@ -678,10 +736,10 @@ public class Agent implements HandlerFactory, IAgentControl { if (agentFile == null) { return new Answer(cmd, false, "Failed to find agent.properties file"); } - final String keyStoreFile = agentFile.getParent() + KeyStoreUtils.defaultKeystoreFile; - final String certFile = agentFile.getParent() + KeyStoreUtils.defaultCertFile; - final String privateKeyFile = agentFile.getParent() + KeyStoreUtils.defaultPrivateKeyFile; - final String caCertFile = agentFile.getParent() + KeyStoreUtils.defaultCaCertFile; + final String keyStoreFile = agentFile.getParent() + KeyStoreUtils.KS_FILENAME; + final String certFile = agentFile.getParent() + KeyStoreUtils.CERT_FILENAME; + final String privateKeyFile = agentFile.getParent() + KeyStoreUtils.PKEY_FILENAME; + final String caCertFile = agentFile.getParent() + KeyStoreUtils.CACERT_FILENAME; try { FileUtils.writeStringToFile(new File(certFile), certificate, Charset.defaultCharset()); @@ -694,7 +752,7 @@ public class Agent implements HandlerFactory, IAgentControl { Script script = new Script(true, _keystoreCertImportPath, 60000, s_logger); script.add(agentFile.getAbsolutePath()); script.add(keyStoreFile); - script.add(KeyStoreUtils.agentMode); + script.add(KeyStoreUtils.AGENT_MODE); script.add(certFile); script.add(""); script.add(caCertFile); @@ -1044,6 +1102,60 @@ public class Agent implements HandlerFactory, IAgentControl { } } + /** + * Task stops the current agent and launches a new agent + * when there are no outstanding jobs in the agent's task queue + */ + public class PostCertificateRenewalTask extends ManagedContextTimerTask { + + private Agent agent; + + public PostCertificateRenewalTask(final Agent agent) { + this.agent = agent; + } + + @Override + protected void runInContext() { + while (true) { + try { + if (_inProgress.get() == 0) { + s_logger.debug("Running post certificate renewal task to restart services."); + + // Let the resource perform any post certificate renewal cleanups + _resource.executeRequest(new PostCertificateRenewalCommand()); + + IAgentShell shell = agent._shell; + ServerResource resource = agent._resource.getClass().newInstance(); + + // Stop current agent + agent.cancelTasks(); + agent._reconnectAllowed = false; + Runtime.getRuntime().removeShutdownHook(agent._shutdownThread); + agent.stop(ShutdownCommand.Requested, "Restarting due to new X509 certificates"); + + // Nullify references for GC + agent._shell = null; + agent._watchList = null; + agent._shutdownThread = null; + agent._controlListeners = null; + agent = null; + + // Start a new agent instance + shell.launchNewAgent(resource); + return; + } + if (s_logger.isTraceEnabled()) { + s_logger.debug("Other tasks are in progress, will retry post certificate renewal command after few seconds"); + } + Thread.sleep(5000); + } catch (final Exception e) { + s_logger.warn("Failed to execute post certificate renewal command:", e); + break; + } + } + } + } + public class PreferredHostCheckerTask extends ManagedContextTimerTask { @Override diff --git a/agent/src/com/cloud/agent/AgentShell.java b/agent/src/com/cloud/agent/AgentShell.java index d06b157605f..0d6f83fd583 100644 --- a/agent/src/com/cloud/agent/AgentShell.java +++ b/agent/src/com/cloud/agent/AgentShell.java @@ -418,7 +418,7 @@ public class AgentShell implements IAgentShell, Daemon { final Constructor constructor = impl.getDeclaredConstructor(); constructor.setAccessible(true); ServerResource resource = (ServerResource)constructor.newInstance(); - launchAgent(getNextAgentId(), resource); + launchNewAgent(resource); } catch (final ClassNotFoundException e) { throw new ConfigurationException("Resource class not found: " + name + " due to: " + e.toString()); } catch (final SecurityException e) { @@ -446,9 +446,10 @@ public class AgentShell implements IAgentShell, Daemon { s_logger.trace("Launching agent based on type=" + typeInfo); } - private void launchAgent(int localAgentId, ServerResource resource) throws ConfigurationException { + public void launchNewAgent(ServerResource resource) throws ConfigurationException { // we don't track agent after it is launched for now - Agent agent = new Agent(this, localAgentId, resource); + _agents.clear(); + Agent agent = new Agent(this, getNextAgentId(), resource); _agents.add(agent); agent.start(); } diff --git a/agent/src/com/cloud/agent/IAgentShell.java b/agent/src/com/cloud/agent/IAgentShell.java index 5b52cee6361..5d389a07041 100644 --- a/agent/src/com/cloud/agent/IAgentShell.java +++ b/agent/src/com/cloud/agent/IAgentShell.java @@ -19,6 +19,9 @@ package com.cloud.agent; import java.util.Map; import java.util.Properties; +import javax.naming.ConfigurationException; + +import com.cloud.resource.ServerResource; import com.cloud.utils.backoff.BackoffAlgorithm; public interface IAgentShell { @@ -66,4 +69,6 @@ public interface IAgentShell { void updateConnectedHost(); String getConnectedHost(); + + void launchNewAgent(ServerResource resource) throws ConfigurationException; } diff --git a/client/WEB-INF/classes/resources/messages.properties b/client/WEB-INF/classes/resources/messages.properties index cc522382862..91d276d41ad 100644 --- a/client/WEB-INF/classes/resources/messages.properties +++ b/client/WEB-INF/classes/resources/messages.properties @@ -278,6 +278,7 @@ label.action.stop.router.processing=Stopping Router.... label.action.stop.router=Stop Router label.action.stop.systemvm.processing=Stopping System VM.... label.action.stop.systemvm=Stop System VM +label.action.secure.host=Provision Host Security Keys label.action.take.snapshot.processing=Taking Snapshot.... label.action.take.snapshot=Take Snapshot label.action.revert.snapshot.processing=Reverting to Snapshot... @@ -1747,6 +1748,7 @@ message.action.start.systemvm=Please confirm that you want to start this system message.action.stop.instance=Please confirm that you want to stop this instance. message.action.stop.router=All services provided by this virtual router will be interrupted. Please confirm that you want to stop this router. message.action.stop.systemvm=Please confirm that you want to stop this system VM. +message.action.secure.host=This will restart the host agent and libvirtd process after applying new X509 certificates, please confirm? message.action.take.snapshot=Please confirm that you want to take a snapshot of this volume. message.action.revert.snapshot=Please confirm that you want to revert the owning volume to this snapshot. message.action.unmanage.cluster=Please confirm that you want to unmanage the cluster. diff --git a/core/src/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java b/core/src/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java index 94e790194a5..a62db2ccf02 100755 --- a/core/src/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java +++ b/core/src/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java @@ -154,11 +154,11 @@ public class VirtualRoutingResource { "/usr/local/cloud/systemvm/conf/%s " + "%s %d " + "/usr/local/cloud/systemvm/conf/%s", - KeyStoreUtils.defaultKeystoreFile, + KeyStoreUtils.KS_FILENAME, cmd.getKeystorePassword(), cmd.getValidityDays(), - KeyStoreUtils.defaultCsrFile); - ExecutionResult result = _vrDeployer.executeInVR(cmd.getRouterAccessIp(), KeyStoreUtils.keyStoreSetupScript, args); + KeyStoreUtils.CSR_FILENAME); + ExecutionResult result = _vrDeployer.executeInVR(cmd.getRouterAccessIp(), KeyStoreUtils.KS_SETUP_SCRIPT, args); return new SetupKeystoreAnswer(result.getDetails()); } @@ -168,15 +168,15 @@ public class VirtualRoutingResource { "/usr/local/cloud/systemvm/conf/%s \"%s\" " + "/usr/local/cloud/systemvm/conf/%s \"%s\" " + "/usr/local/cloud/systemvm/conf/%s \"%s\"", - KeyStoreUtils.defaultKeystoreFile, - KeyStoreUtils.sshMode, - KeyStoreUtils.defaultCertFile, + KeyStoreUtils.KS_FILENAME, + KeyStoreUtils.SSH_MODE, + KeyStoreUtils.CERT_FILENAME, cmd.getEncodedCertificate(), - KeyStoreUtils.defaultCaCertFile, + KeyStoreUtils.CACERT_FILENAME, cmd.getEncodedCaCertificates(), - KeyStoreUtils.defaultPrivateKeyFile, + KeyStoreUtils.PKEY_FILENAME, cmd.getEncodedPrivateKey()); - ExecutionResult result = _vrDeployer.executeInVR(cmd.getRouterAccessIp(), KeyStoreUtils.keyStoreImportScript, args); + ExecutionResult result = _vrDeployer.executeInVR(cmd.getRouterAccessIp(), KeyStoreUtils.KS_IMPORT_SCRIPT, args); return new SetupCertificateAnswer(result.isSuccess()); } diff --git a/core/src/org/apache/cloudstack/ca/PostCertificateRenewalCommand.java b/core/src/org/apache/cloudstack/ca/PostCertificateRenewalCommand.java new file mode 100644 index 00000000000..12df6196128 --- /dev/null +++ b/core/src/org/apache/cloudstack/ca/PostCertificateRenewalCommand.java @@ -0,0 +1,34 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.ca; + +import com.cloud.agent.api.Command; + +public class PostCertificateRenewalCommand extends Command { + + public PostCertificateRenewalCommand() { + super(); + } + + @Override + public boolean executeInSequence() { + return false; + } +} diff --git a/core/src/org/apache/cloudstack/ca/SetupCertificateCommand.java b/core/src/org/apache/cloudstack/ca/SetupCertificateCommand.java index 1cd31509d39..7727282bcee 100644 --- a/core/src/org/apache/cloudstack/ca/SetupCertificateCommand.java +++ b/core/src/org/apache/cloudstack/ca/SetupCertificateCommand.java @@ -82,15 +82,15 @@ public class SetupCertificateCommand extends NetworkElementCommand { } public String getEncodedPrivateKey() { - return privateKey.replace("\n", KeyStoreUtils.certNewlineEncoder).replace(" ", KeyStoreUtils.certSpaceEncoder); + return privateKey.replace("\n", KeyStoreUtils.CERT_NEWLINE_ENCODER).replace(" ", KeyStoreUtils.CERT_SPACE_ENCODER); } public String getEncodedCertificate() { - return certificate.replace("\n", KeyStoreUtils.certNewlineEncoder).replace(" ", KeyStoreUtils.certSpaceEncoder); + return certificate.replace("\n", KeyStoreUtils.CERT_NEWLINE_ENCODER).replace(" ", KeyStoreUtils.CERT_SPACE_ENCODER); } public String getEncodedCaCertificates() { - return caCertificates.replace("\n", KeyStoreUtils.certNewlineEncoder).replace(" ", KeyStoreUtils.certSpaceEncoder); + return caCertificates.replace("\n", KeyStoreUtils.CERT_NEWLINE_ENCODER).replace(" ", KeyStoreUtils.CERT_SPACE_ENCODER); } public boolean isHandleByAgent() { diff --git a/debian/cloudstack-agent.postinst b/debian/cloudstack-agent.postinst index 01aaef60a67..b00f3f1f1a0 100644 --- a/debian/cloudstack-agent.postinst +++ b/debian/cloudstack-agent.postinst @@ -41,6 +41,14 @@ case "$1" in mkdir /etc/libvirt/hooks fi cp -a /usr/share/cloudstack-agent/lib/libvirtqemuhook /etc/libvirt/hooks/qemu + + # Enable libvirt TLS if host is secured + if [ -f "/etc/cloudstack/agent/cloud.jks" ]; then + /usr/bin/cloudstack-setup-agent -s + /sbin/service libvirt-bin restart + /sbin/iptables -t filter -A INPUT -p tcp -m tcp --dport 16514 -j ACCEPT + /sbin/iptables-save > /etc/iptables/rules.v4 + fi ;; esac diff --git a/packaging/centos63/cloud.spec b/packaging/centos63/cloud.spec index 9c7f3a41c9a..730124ee750 100644 --- a/packaging/centos63/cloud.spec +++ b/packaging/centos63/cloud.spec @@ -557,6 +557,14 @@ if [ -f "%{_sysconfdir}/cloud.rpmsave/agent/agent.properties" ]; then mv %{_sysconfdir}/cloud.rpmsave/agent/agent.properties %{_sysconfdir}/cloud.rpmsave/agent/agent.properties.rpmsave fi +# Enable libvirt TLS if host is secured +if [ -f "/etc/cloudstack/agent/cloud.jks" ]; then + /usr/bin/cloudstack-setup-agent -s + /sbin/service libvirtd restart + /sbin/iptables -t filter -A INPUT -p tcp -m tcp --dport 16514 -j ACCEPT + /sbin/service iptables save +fi + %preun usage /sbin/service cloudstack-usage stop || true if [ "$1" == "0" ] ; then diff --git a/plugins/ca/root-ca/src/org/apache/cloudstack/ca/provider/RootCAProvider.java b/plugins/ca/root-ca/src/org/apache/cloudstack/ca/provider/RootCAProvider.java index 76e706963e6..911274886d6 100644 --- a/plugins/ca/root-ca/src/org/apache/cloudstack/ca/provider/RootCAProvider.java +++ b/plugins/ca/root-ca/src/org/apache/cloudstack/ca/provider/RootCAProvider.java @@ -335,7 +335,7 @@ public final class RootCAProvider extends AdapterBase implements CAProvider, Con } private char[] getCaKeyStorePassphrase() { - return KeyStoreUtils.defaultKeystorePassphrase; + return KeyStoreUtils.DEFAULT_KS_PASSPHRASE; } private KeyStore getCaKeyStore() throws CertificateException, NoSuchAlgorithmException, IOException, KeyStoreException { @@ -373,7 +373,7 @@ public final class RootCAProvider extends AdapterBase implements CAProvider, Con ////////////////////////////////////////////////// private char[] findKeyStorePassphrase() { - char[] passphrase = KeyStoreUtils.defaultKeystorePassphrase; + char[] passphrase = KeyStoreUtils.DEFAULT_KS_PASSPHRASE; final String configuredPassphrase = DbProperties.getDbProperties().getProperty("db.cloud.keyStorePassphrase"); if (configuredPassphrase != null) { passphrase = configuredPassphrase.toCharArray(); @@ -394,7 +394,7 @@ public final class RootCAProvider extends AdapterBase implements CAProvider, Con keyStore.setCertificateEntry(caAlias, caCertificate); keyStore.setKeyEntry(managementAlias, managementServerCertificate.getPrivateKey(), passphrase, new X509Certificate[]{managementServerCertificate.getClientCertificate(), caCertificate}); - final String tmpFile = KeyStoreUtils.defaultTmpKeyStoreFile; + final String tmpFile = KeyStoreUtils.KS_TMP_FILE; final FileOutputStream stream = new FileOutputStream(tmpFile); keyStore.store(stream, passphrase); stream.close(); @@ -413,7 +413,7 @@ public final class RootCAProvider extends AdapterBase implements CAProvider, Con return false; } final char[] passphrase = findKeyStorePassphrase(); - final String keystorePath = confFile.getParent() + KeyStoreUtils.defaultKeystoreFile; + final String keystorePath = confFile.getParent() + KeyStoreUtils.KS_FILENAME; final File keystoreFile = new File(keystorePath); if (keystoreFile.exists()) { try { diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 9a3c45dbc3c..d26dfc998e1 100755 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -61,8 +61,19 @@ import java.util.regex.Pattern; import javax.ejb.Local; import javax.naming.ConfigurationException; +import org.apache.cloudstack.ca.PostCertificateRenewalCommand; +import org.apache.cloudstack.ca.SetupCertificateAnswer; +import org.apache.cloudstack.storage.command.StorageSubSystemCommand; +import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; +import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.apache.cloudstack.utils.hypervisor.HypervisorUtils; import org.apache.cloudstack.utils.linux.CPUStat; import org.apache.cloudstack.utils.linux.MemStat; +import org.apache.cloudstack.utils.qemu.QemuImg; +import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; +import org.apache.cloudstack.utils.qemu.QemuImgException; +import org.apache.cloudstack.utils.qemu.QemuImgFile; +import org.apache.cloudstack.utils.security.KeyStoreUtils; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.log4j.Logger; @@ -83,16 +94,6 @@ import com.ceph.rados.RadosException; import com.ceph.rbd.Rbd; import com.ceph.rbd.RbdException; import com.ceph.rbd.RbdImage; - -import org.apache.cloudstack.storage.command.StorageSubSystemCommand; -import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; -import org.apache.cloudstack.storage.to.VolumeObjectTO; -import org.apache.cloudstack.utils.qemu.QemuImg; -import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; -import org.apache.cloudstack.utils.qemu.QemuImgException; -import org.apache.cloudstack.utils.qemu.QemuImgFile; -import org.apache.cloudstack.utils.hypervisor.HypervisorUtils; - import com.cloud.agent.api.Answer; import com.cloud.agent.api.AttachIsoCommand; import com.cloud.agent.api.AttachVolumeAnswer; @@ -237,8 +238,8 @@ import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.InterfaceDef; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.InterfaceDef.guestNetType; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.SerialDef; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.TermPolicy; -import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.VirtioSerialDef; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.VideoDef; +import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.VirtioSerialDef; import com.cloud.hypervisor.kvm.storage.KVMPhysicalDisk; import com.cloud.hypervisor.kvm.storage.KVMStoragePool; import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; @@ -276,6 +277,7 @@ import com.cloud.utils.ssh.SshHelper; import com.cloud.vm.DiskProfile; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachine.PowerState; +import com.google.common.base.Strings; /** * LibvirtComputingResource execute requests on the computing/routing host using @@ -466,6 +468,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv protected boolean _noKvmClock; protected String _videoHw; protected int _videoRam; + protected String _hostDistro; protected Pair hostOsVersion; private final Map _pifs = new HashMap(); private final Map _vmStats = new ConcurrentHashMap(); @@ -1308,6 +1311,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv try { if (cmd instanceof StopCommand) { return execute((StopCommand)cmd); + } else if (cmd instanceof PostCertificateRenewalCommand) { + return execute((PostCertificateRenewalCommand) cmd); } else if (cmd instanceof GetVmStatsCommand) { return execute((GetVmStatsCommand)cmd); } else if (cmd instanceof GetVmDiskStatsCommand) { @@ -3106,6 +3111,13 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv return command.execute(); } + protected String createMigrationURI(final String destinationIp) { + if (Strings.isNullOrEmpty(destinationIp)) { + throw new CloudRuntimeException("Provided libvirt destination ip is invalid"); + } + return String.format("%s://%s/system", KeyStoreUtils.isHostSecured() ? "qemu+tls" : "qemu+tcp", destinationIp); + } + private Answer execute(MigrateCommand cmd) { String vmName = cmd.getVmName(); @@ -3114,6 +3126,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv List ifaces = null; List disks = null; + final String destinationUri = createMigrationURI(cmd.getDestinationIp()); Domain dm = null; Connect dconn = null; Domain destDomain = null; @@ -3148,10 +3161,10 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv xmlDesc = dm.getXMLDesc(xmlFlag).replace(_privateIp, cmd.getDestinationIp()); - dconn = new Connect("qemu+tcp://" + cmd.getDestinationIp() + "/system"); + dconn = new Connect(destinationUri); //run migration in thread so we can monitor it - s_logger.info("Live migration of instance " + vmName + " initiated"); + s_logger.info("Live migration of instance " + vmName + " initiated to destination host: " + dconn.getURI()); ExecutorService executor = Executors.newFixedThreadPool(1); Callable worker = new MigrateKVMAsync(dm, dconn, xmlDesc, vmName, cmd.getDestinationIp()); Future migrateThread = executor.submit(worker); @@ -3199,6 +3212,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv } catch (LibvirtException e) { s_logger.debug("Can't migrate domain: " + e.getMessage()); result = e.getMessage(); + if (result.startsWith("unable to connect to server") && result.endsWith("refused")) { + result = String.format("Migration was refused connection to destination: %s. Please check libvirt configuration compatibility on the source and destination hosts.", destinationUri); + } } catch (InterruptedException e) { s_logger.debug("Interrupted while migrating domain: " + e.getMessage()); result = e.getMessage(); @@ -3554,6 +3570,23 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv } } + protected Answer execute(final PostCertificateRenewalCommand cmd) { + s_logger.info("Restarting libvirt after certificate provisioning/renewal"); + if (cmd != null) { + final int timeout = 30000; + Script script = new Script(true, "service", timeout, s_logger); + if ("Ubuntu".equals(_hostDistro) || "Debian".equals(_hostDistro)) { + script.add("libvirt-bin"); + } else { + script.add("libvirtd"); + } + script.add("restart"); + script.execute(); + return new SetupCertificateAnswer(true); + } + return new SetupCertificateAnswer(false); + } + protected Answer execute(ModifySshKeysCommand cmd) { File sshKeysDir = new File(SSHKEYSPATH); String result = null; @@ -4303,11 +4336,16 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv fillNetworkInformation(cmd); _privateIp = cmd.getPrivateIpAddress(); cmd.getHostDetails().putAll(getVersionStrings()); + cmd.getHostDetails().put(KeyStoreUtils.SECURED, String.valueOf(KeyStoreUtils.isHostSecured()).toLowerCase()); cmd.setPool(_pool); cmd.setCluster(_clusterId); cmd.setGatewayIpAddress(_localGateway); cmd.setIqn(getIqn()); + if (cmd.getHostDetails().containsKey("Host.OS")) { + _hostDistro = cmd.getHostDetails().get("Host.OS"); + } + StartupStorageCommand sscmd = null; try { diff --git a/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java b/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java index bf4546c63d1..0bb32b66eff 100644 --- a/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java +++ b/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java @@ -27,8 +27,7 @@ import java.util.List; import java.util.Random; import java.util.UUID; -import junit.framework.Assert; - +import org.apache.cloudstack.utils.security.KeyStoreUtils; import org.apache.commons.lang.SystemUtils; import org.junit.Assume; import org.junit.Test; @@ -50,8 +49,11 @@ import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.DiskDef; import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.InterfaceDef; import com.cloud.template.VirtualMachineTemplate.BootloaderType; import com.cloud.utils.Pair; +import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VirtualMachine; +import junit.framework.Assert; + public class LibvirtComputingResourceTest { String _hyperVisorType = "kvm"; @@ -454,4 +456,21 @@ public class LibvirtComputingResourceTest { NodeInfo nodeInfo = Mockito.mock(NodeInfo.class); LibvirtComputingResource.getCpuSpeed(nodeInfo); } + + @Test + public void testMigrationUri() { + final String ip = "10.1.1.1"; + LibvirtComputingResource lcr = new LibvirtComputingResource(); + if (KeyStoreUtils.isHostSecured()) { + Assert.assertEquals(lcr.createMigrationURI(ip), String.format("qemu+tls://%s/system", ip)); + } else { + Assert.assertEquals(lcr.createMigrationURI(ip), String.format("qemu+tcp://%s/system", ip)); + } + } + + @Test(expected = CloudRuntimeException.class) + public void testMigrationUriException() { + LibvirtComputingResource lcr = new LibvirtComputingResource(); + lcr.createMigrationURI(null); + } } diff --git a/python/lib/cloud_utils.py b/python/lib/cloud_utils.py index b01cd9c1a23..74a4f4b9e82 100644 --- a/python/lib/cloud_utils.py +++ b/python/lib/cloud_utils.py @@ -804,7 +804,7 @@ class SetupFirewall(ConfigTask): return False def execute(self): - ports = "22 1798 16509".split() + ports = "22 1798 16509 16514".split() if distro in (Fedora , CentOS, RHEL6): for p in ports: iptables("-I","INPUT","1","-p","tcp","--dport",p,'-j','ACCEPT') o = service.iptables.save() ; print o.stdout + o.stderr diff --git a/python/lib/cloudutils/serviceConfig.py b/python/lib/cloudutils/serviceConfig.py index 292c9a77d76..5cd258e6cee 100755 --- a/python/lib/cloudutils/serviceConfig.py +++ b/python/lib/cloudutils/serviceConfig.py @@ -471,6 +471,23 @@ class securityPolicyConfigRedhat(serviceCfgBase): logging.debug(formatExceptionInfo()) return False +def configureLibvirtConfig(tls_enabled = True, cfg = None): + cfo = configFileOps("/etc/libvirt/libvirtd.conf", cfg) + if tls_enabled: + cfo.addEntry("listen_tcp", "0") + cfo.addEntry("listen_tls", "1") + cfo.addEntry("key_file", "\"/etc/pki/libvirt/private/serverkey.pem\"") + cfo.addEntry("cert_file", "\"/etc/pki/libvirt/servercert.pem\"") + cfo.addEntry("ca_file", "\"/etc/pki/CA/cacert.pem\"") + else: + cfo.addEntry("listen_tcp", "1") + cfo.addEntry("listen_tls", "0") + cfo.addEntry("tcp_port", "\"16509\"") + cfo.addEntry("tls_port", "\"16514\"") + cfo.addEntry("auth_tcp", "\"none\"") + cfo.addEntry("auth_tls", "\"none\"") + cfo.save() + class libvirtConfigRedhat(serviceCfgBase): def __init__(self, syscfg): super(libvirtConfigRedhat, self).__init__(syscfg) @@ -478,12 +495,7 @@ class libvirtConfigRedhat(serviceCfgBase): def config(self): try: - cfo = configFileOps("/etc/libvirt/libvirtd.conf", self) - cfo.addEntry("listen_tcp", "1") - cfo.addEntry("tcp_port", "\"16509\"") - cfo.addEntry("auth_tcp", "\"none\"") - cfo.addEntry("listen_tls", "0") - cfo.save() + configureLibvirtConfig(self.syscfg.env.secure, self) cfo = configFileOps("/etc/sysconfig/libvirtd", self) cfo.addEntry("export CGROUP_DAEMON", "'cpu:/virt'") @@ -515,24 +527,16 @@ class libvirtConfigUbuntu(serviceCfgBase): super(libvirtConfigUbuntu, self).__init__(syscfg) self.serviceName = "Libvirt" - def setupLiveMigration(self): - cfo = configFileOps("/etc/libvirt/libvirtd.conf", self) - cfo.addEntry("listen_tcp", "1") - cfo.addEntry("tcp_port", "\"16509\""); - cfo.addEntry("auth_tcp", "\"none\""); - cfo.addEntry("listen_tls", "0") - cfo.save() - - if os.path.exists("/etc/init/libvirt-bin.conf"): - cfo = configFileOps("/etc/init/libvirt-bin.conf", self) - cfo.replace_line("exec /usr/sbin/libvirtd","exec /usr/sbin/libvirtd -d -l") - else: - cfo = configFileOps("/etc/default/libvirt-bin", self) - cfo.replace_or_add_line("libvirtd_opts=","libvirtd_opts='-l -d'") - def config(self): try: - self.setupLiveMigration() + configureLibvirtConfig(self.syscfg.env.secure, self) + if os.path.exists("/etc/init/libvirt-bin.conf"): + cfo = configFileOps("/etc/init/libvirt-bin.conf", self) + cfo.replace_line("exec /usr/sbin/libvirtd","exec /usr/sbin/libvirtd -d -l") + else: + cfo = configFileOps("/etc/default/libvirt-bin", self) + cfo.replace_or_add_line("libvirtd_opts=","libvirtd_opts='-l -d'") + filename = "/etc/libvirt/qemu.conf" @@ -564,7 +568,7 @@ class firewallConfigUbuntu(serviceCfgBase): def config(self): try: - ports = "22 1798 16509".split() + ports = "22 1798 16509 16514".split() for p in ports: bash("ufw allow %s"%p) bash("ufw allow proto tcp from any to any port 5900:6100") @@ -624,7 +628,7 @@ class firewallConfigBase(serviceCfgBase): class firewallConfigAgent(firewallConfigBase): def __init__(self, syscfg): super(firewallConfigAgent, self).__init__(syscfg) - self.ports = "22 16509 5900:6100 49152:49216".split() + self.ports = "22 16509 16514 5900:6100 49152:49216".split() if syscfg.env.distribution.getVersion() == "CentOS": self.rules = ["-D FORWARD -j RH-Firewall-1-INPUT"] else: diff --git a/scripts/util/keystore-cert-import b/scripts/util/keystore-cert-import index bb03b6f68b4..d2adc6efb2e 100755 --- a/scripts/util/keystore-cert-import +++ b/scripts/util/keystore-cert-import @@ -28,6 +28,7 @@ PRIVKEY=$(echo "$9" | tr '^' '\n' | tr '~' ' ') ALIAS="cloud" SYSTEM_FILE="/var/cache/cloud/cmdline" +LIBVIRTD_FILE="/etc/libvirt/libvirtd.conf" # Find keystore password KS_PASS=$(sed -n '/keystore.passphrase/p' "$PROPS_FILE" 2>/dev/null | sed 's/keystore.passphrase=//g' 2>/dev/null) @@ -87,6 +88,17 @@ if [ -f "$SYSTEM_FILE" ]; then update-ca-certificates > /dev/null 2>&1 || true fi +# Secure libvirtd on cert import +if [ -f "$LIBVIRTD_FILE" ]; then + mkdir -p /etc/pki/libvirt/private + ln -sf /etc/cloudstack/agent/cloud.ca.crt /etc/pki/CA/cacert.pem + ln -sf /etc/cloudstack/agent/cloud.crt /etc/pki/libvirt/clientcert.pem + ln -sf /etc/cloudstack/agent/cloud.crt /etc/pki/libvirt/servercert.pem + ln -sf /etc/cloudstack/agent/cloud.key /etc/pki/libvirt/private/clientkey.pem + ln -sf /etc/cloudstack/agent/cloud.key /etc/pki/libvirt/private/serverkey.pem + cloudstack-setup-agent -s > /dev/null +fi + # Restart cloud service if we're in systemvm if [ "$MODE" == "ssh" ] && [ -f $SYSTEM_FILE ]; then /etc/init.d/cloud stop > /dev/null 2>&1 diff --git a/scripts/util/keystore-setup b/scripts/util/keystore-setup index 28ce61c846a..48ce06220ca 100755 --- a/scripts/util/keystore-setup +++ b/scripts/util/keystore-setup @@ -38,11 +38,11 @@ fi # Generate keystore rm -f "$KS_FILE" CN=$(hostname --fqdn) -keytool -genkey -storepass "$KS_PASS" -keypass "$KS_PASS" -alias "$ALIAS" -keyalg RSA -validity "$KS_VALIDITY" -dname cn="$CN",ou="cloudstack",o="cloudstack",c="cloudstack" -keystore "$KS_FILE" +keytool -genkey -storepass "$KS_PASS" -keypass "$KS_PASS" -alias "$ALIAS" -keyalg RSA -validity "$KS_VALIDITY" -dname cn="$CN",ou="cloudstack",o="cloudstack",c="cloudstack" -keystore "$KS_FILE" > /dev/null 2>&1 # Generate CSR rm -f "$CSR_FILE" -keytool -certreq -storepass "$KS_PASS" -alias "$ALIAS" -file $CSR_FILE -keystore "$KS_FILE" +keytool -certreq -storepass "$KS_PASS" -alias "$ALIAS" -file $CSR_FILE -keystore "$KS_FILE" > /dev/null 2>&1 cat "$CSR_FILE" # Fix file permissions diff --git a/server/src/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java b/server/src/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java index eb2628e6e84..fc7186bf03a 100644 --- a/server/src/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java +++ b/server/src/com/cloud/hypervisor/kvm/discoverer/LibvirtServerDiscoverer.java @@ -18,6 +18,7 @@ package com.cloud.hypervisor.kvm.discoverer; import java.net.InetAddress; import java.net.URI; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -27,6 +28,7 @@ import java.util.UUID; import javax.inject.Inject; import javax.naming.ConfigurationException; +import org.apache.cloudstack.agent.lb.IndirectAgentLB; import org.apache.cloudstack.ca.CAManager; import org.apache.cloudstack.ca.SetupCertificateCommand; import org.apache.cloudstack.framework.ca.Certificate; @@ -64,7 +66,6 @@ import com.cloud.utils.StringUtils; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.ssh.SSHCmdHelper; import com.trilead.ssh2.Connection; -import org.apache.cloudstack.agent.lb.IndirectAgentLB; public abstract class LibvirtServerDiscoverer extends DiscovererBase implements Discoverer, Listener, ResourceStateAdapter { private static final Logger s_logger = Logger.getLogger(LibvirtServerDiscoverer.class); @@ -130,11 +131,6 @@ public abstract class LibvirtServerDiscoverer extends DiscovererBase implements } private void setupAgentSecurity(final Connection sshConnection, final String agentIp, final String agentHostname) { - if (!caManager.canProvisionCertificates()) { - s_logger.warn("Cannot secure agent communication because configure CA plugin cannot provision client certificate"); - return; - } - if (sshConnection == null) { throw new CloudRuntimeException("Cannot secure agent communication because ssh connection is invalid for host ip=" + agentIp); } @@ -150,17 +146,17 @@ public abstract class LibvirtServerDiscoverer extends DiscovererBase implements "/etc/cloudstack/agent/%s " + "%s %d " + "/etc/cloudstack/agent/%s", - KeyStoreUtils.keyStoreSetupScript, - KeyStoreUtils.defaultKeystoreFile, + KeyStoreUtils.KS_SETUP_SCRIPT, + KeyStoreUtils.KS_FILENAME, PasswordGenerator.generateRandomPassword(16), validityPeriod, - KeyStoreUtils.defaultCsrFile)); + KeyStoreUtils.CSR_FILENAME)); if (!keystoreSetupResult.isSuccess()) { throw new CloudRuntimeException("Failed to setup keystore on the KVM host: " + agentIp); } - final Certificate certificate = caManager.issueCertificate(keystoreSetupResult.getStdOut(), Collections.singletonList(agentHostname), Collections.singletonList(agentIp), null, null); + final Certificate certificate = caManager.issueCertificate(keystoreSetupResult.getStdOut(), Arrays.asList(agentHostname, agentIp), Collections.singletonList(agentIp), null, null); if (certificate == null || certificate.getClientCertificate() == null) { throw new CloudRuntimeException("Failed to issue certificates for KVM host agent: " + agentIp); } @@ -173,14 +169,14 @@ public abstract class LibvirtServerDiscoverer extends DiscovererBase implements "/etc/cloudstack/agent/%s \"%s\" " + "/etc/cloudstack/agent/%s \"%s\" " + "/etc/cloudstack/agent/%s \"%s\"", - KeyStoreUtils.keyStoreImportScript, - KeyStoreUtils.defaultKeystoreFile, - KeyStoreUtils.sshMode, - KeyStoreUtils.defaultCertFile, + KeyStoreUtils.KS_IMPORT_SCRIPT, + KeyStoreUtils.KS_FILENAME, + KeyStoreUtils.SSH_MODE, + KeyStoreUtils.CERT_FILENAME, certificateCommand.getEncodedCertificate(), - KeyStoreUtils.defaultCaCertFile, + KeyStoreUtils.CACERT_FILENAME, certificateCommand.getEncodedCaCertificates(), - KeyStoreUtils.defaultPrivateKeyFile, + KeyStoreUtils.PKEY_FILENAME, certificateCommand.getEncodedPrivateKey())); if (setupCertResult != null && !setupCertResult.isSuccess()) { @@ -277,9 +273,13 @@ public abstract class LibvirtServerDiscoverer extends DiscovererBase implements kvmGuestNic = (kvmPublicNic != null) ? kvmPublicNic : kvmPrivateNic; } + if (!caManager.canProvisionCertificates()) { + throw new CloudRuntimeException("Configured CA plugin cannot provision X509 certificate(s), failing to add host due to security insufficiency."); + } + setupAgentSecurity(sshConnection, agentIp, hostname); - String parameters = " -m " + StringUtils.toCSVList(indirectAgentLB.getManagementServerList(null, dcId, null)) + " -z " + dcId + " -p " + podId + " -c " + clusterId + " -g " + guid + " -a"; + String parameters = " -m " + StringUtils.toCSVList(indirectAgentLB.getManagementServerList(null, dcId, null)) + " -z " + dcId + " -p " + podId + " -c " + clusterId + " -g " + guid + " -a -s"; parameters += " --pubNic=" + kvmPublicNic; parameters += " --prvNic=" + kvmPrivateNic; diff --git a/server/src/org/apache/cloudstack/ca/CAManagerImpl.java b/server/src/org/apache/cloudstack/ca/CAManagerImpl.java index b3fb259f76c..5cc1435dbae 100644 --- a/server/src/org/apache/cloudstack/ca/CAManagerImpl.java +++ b/server/src/org/apache/cloudstack/ca/CAManagerImpl.java @@ -25,7 +25,6 @@ import java.security.cert.CertificateNotYetValidException; import java.security.cert.X509Certificate; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.Iterator; @@ -188,7 +187,7 @@ public class CAManagerImpl extends ManagerBase implements CAManager { if (Strings.isNullOrEmpty(csr)) { return false; } - final Certificate certificate = issueCertificate(csr, Collections.singletonList(host.getName()), Arrays.asList(host.getPrivateIpAddress(), host.getPublicIpAddress(), host.getStorageIpAddress()), CAManager.CertValidityPeriod.value(), caProvider); + final Certificate certificate = issueCertificate(csr, Arrays.asList(host.getName(), host.getPrivateIpAddress()), Arrays.asList(host.getPrivateIpAddress(), host.getPublicIpAddress(), host.getStorageIpAddress()), CAManager.CertValidityPeriod.value(), caProvider); return deployCertificate(host, certificate, reconnect, null); } catch (final AgentUnavailableException | OperationTimedoutException e) { LOG.error("Host/agent is not available or operation timed out, failed to setup keystore and generate CSR for host/agent id=" + host.getId() + ", due to: ", e); diff --git a/ui/css/cloudstack3.css b/ui/css/cloudstack3.css index b9ae0eb4370..79c0c95636b 100644 --- a/ui/css/cloudstack3.css +++ b/ui/css/cloudstack3.css @@ -12562,11 +12562,13 @@ div.ui-dialog div.autoscaler div.field-group div.form-container form div.form-it background-position: -101px -647px; } +.secureKVMHost .icon, .resetPassword .icon, .changePassword .icon { background-position: -68px -30px; } +.secureKVMHost:hover .icon, .resetPassword:hover .icon, .changePassword:hover .icon { background-position: -68px -612px; diff --git a/ui/dictionary.jsp b/ui/dictionary.jsp index 4f1d0b2c84f..92943beabd9 100644 --- a/ui/dictionary.jsp +++ b/ui/dictionary.jsp @@ -306,6 +306,7 @@ dictionary = { 'label.action.stop.router.processing': '', 'label.action.stop.systemvm': '', 'label.action.stop.systemvm.processing': '', +'label.action.secure.host': '', 'label.action.take.snapshot': '', 'label.action.take.snapshot.processing': '', 'label.action.revert.snapshot': '', diff --git a/ui/dictionary2.jsp b/ui/dictionary2.jsp index bd080ae0372..bd9e4f5595e 100644 --- a/ui/dictionary2.jsp +++ b/ui/dictionary2.jsp @@ -398,6 +398,7 @@ $.extend(dictionary, { 'message.action.stop.instance': '', 'message.action.stop.router': '', 'message.action.stop.systemvm': '', +'message.action.secure.host': '', 'message.action.take.snapshot': '', 'message.action.revert.snapshot': '', 'message.action.unmanage.cluster': '', diff --git a/ui/scripts/system.js b/ui/scripts/system.js index 297375af06a..0a9256dbf8c 100644 --- a/ui/scripts/system.js +++ b/ui/scripts/system.js @@ -8806,6 +8806,11 @@ if (host && host.outofbandmanagement) { items[idx].powerstate = host.outofbandmanagement.powerstate; } + + if (host && host.hypervisor == "KVM" && host.state == 'Up' && host.details && host.details["secured"] != 'true') { + items[idx].state = 'Unsecure'; + } + }); } @@ -15142,7 +15147,8 @@ 'Down': 'off', 'Disconnected': 'off', 'Alert': 'off', - 'Error': 'off' + 'Error': 'off', + 'Unsecure': 'warning' } }, powerstate: { @@ -15190,6 +15196,10 @@ if (host && host.outofbandmanagement) { items[idx].powerstate = host.outofbandmanagement.powerstate; } + + if (host && host.hypervisor == "KVM" && host.state == 'Up' && host.details && host.details["secured"] != 'true') { + items[idx].state = 'Unsecure'; + } }); } @@ -15878,8 +15888,42 @@ poll: pollAsyncJobResult } }, - - + + secureKVMHost: { + label: 'label.action.secure.host', + action: function(args) { + var data = { + hostid: args.context.hosts[0].id + }; + $.ajax({ + url: createURL('provisionCertificate'), + data: data, + async: true, + success: function(json) { + args.response.success({ + _custom: { + jobId: json.provisioncertificateresponse.jobid, + getActionFilter: function () { + return hostActionfilter; + } + } + }); + } + }); + }, + messages: { + confirm: function (args) { + return 'message.action.secure.host'; + }, + notification: function (args) { + return 'label.action.secure.host'; + } + }, + notification: { + poll: pollAsyncJobResult + } + }, + enableMaintenanceMode: { label: 'label.action.enable.maintenance.mode', action: function (args) { @@ -21281,9 +21325,14 @@ allowedActions.push("edit"); allowedActions.push("enableMaintenanceMode"); allowedActions.push("disable"); - + if (jsonObj.state != "Disconnected") allowedActions.push("forceReconnect"); + + if (jsonObj.hypervisor == "KVM") { + allowedActions.push("secureKVMHost"); + } + } else if (jsonObj.resourcestate == "ErrorInMaintenance") { allowedActions.push("edit"); allowedActions.push("enableMaintenanceMode"); diff --git a/utils/src/com/cloud/utils/nio/Link.java b/utils/src/com/cloud/utils/nio/Link.java index 023d33ed590..1b34b260a49 100755 --- a/utils/src/com/cloud/utils/nio/Link.java +++ b/utils/src/com/cloud/utils/nio/Link.java @@ -371,7 +371,7 @@ public class Link { return caService.createSSLEngine(sslContext, clientAddress); } s_logger.error("CA service is not configured, by-passing CA manager to create SSL engine"); - char[] passphrase = KeyStoreUtils.defaultKeystorePassphrase; + char[] passphrase = KeyStoreUtils.DEFAULT_KS_PASSPHRASE; final KeyStore ks = loadKeyStore(NioConnection.class.getResourceAsStream("/cloud.keystore"), passphrase); final KeyManagerFactory kmf = KeyManagerFactory.getInstance("SunX509"); final TrustManagerFactory tmf = TrustManagerFactory.getInstance("SunX509"); @@ -390,11 +390,11 @@ public class Link { public static SSLContext initClientSSLContext() throws GeneralSecurityException, IOException { final SSLContext sslContext = SSLUtils.getSSLContext(); - char[] passphrase = KeyStoreUtils.defaultKeystorePassphrase; + char[] passphrase = KeyStoreUtils.DEFAULT_KS_PASSPHRASE; File confFile = PropertiesUtil.findConfigFile("agent.properties"); if (confFile != null) { s_logger.info("Conf file found: " + confFile.getAbsolutePath()); - final String pass = PropertiesUtil.getProperties(confFile).getProperty(KeyStoreUtils.passphrasePropertyName); + final String pass = PropertiesUtil.getProperties(confFile).getProperty(KeyStoreUtils.KS_PASSPHRASE_PROPERTY); if (pass != null) { passphrase = pass.toCharArray(); } @@ -411,7 +411,7 @@ public class Link { InputStream stream = null; if (confFile != null) { final String confPath = confFile.getParent(); - final String keystorePath = confPath + KeyStoreUtils.defaultKeystoreFile; + final String keystorePath = confPath + KeyStoreUtils.KS_FILENAME; if (new File(keystorePath).exists()) { stream = new FileInputStream(keystorePath); } diff --git a/utils/src/com/cloud/utils/script/Script.java b/utils/src/com/cloud/utils/script/Script.java index b8a9256889a..66fada09577 100755 --- a/utils/src/com/cloud/utils/script/Script.java +++ b/utils/src/com/cloud/utils/script/Script.java @@ -184,7 +184,7 @@ public class Script implements Callable { String[] command = _command.toArray(new String[_command.size()]); if (_logger.isDebugEnabled()) { - _logger.debug("Executing: " + buildCommandLine(command).split(KeyStoreUtils.defaultKeystoreFile)[0]); + _logger.debug("Executing: " + buildCommandLine(command).split(KeyStoreUtils.KS_FILENAME)[0]); } try { diff --git a/utils/src/com/cloud/utils/ssh/SSHCmdHelper.java b/utils/src/com/cloud/utils/ssh/SSHCmdHelper.java index 60a27c37605..56780d80177 100644 --- a/utils/src/com/cloud/utils/ssh/SSHCmdHelper.java +++ b/utils/src/com/cloud/utils/ssh/SSHCmdHelper.java @@ -137,7 +137,7 @@ public class SSHCmdHelper { } public static SSHCmdResult sshExecuteCmdOneShot(com.trilead.ssh2.Connection sshConnection, String cmd) throws SshException { - s_logger.debug("Executing cmd: " + cmd.split(KeyStoreUtils.defaultKeystoreFile)[0]); + s_logger.debug("Executing cmd: " + cmd.split(KeyStoreUtils.KS_FILENAME)[0]); Session sshSession = null; try { sshSession = sshConnection.openSession(); @@ -200,7 +200,7 @@ public class SSHCmdHelper { final SSHCmdResult result = new SSHCmdResult(-1, sbStdoutResult.toString(), sbStdErrResult.toString()); if (!Strings.isNullOrEmpty(result.getStdOut()) || !Strings.isNullOrEmpty(result.getStdErr())) { - s_logger.debug("SSH command: " + cmd.split(KeyStoreUtils.defaultKeystoreFile)[0] + "\nSSH command output:" + result.getStdOut().split("-----BEGIN")[0] + "\n" + result.getStdErr()); + s_logger.debug("SSH command: " + cmd.split(KeyStoreUtils.KS_FILENAME)[0] + "\nSSH command output:" + result.getStdOut().split("-----BEGIN")[0] + "\n" + result.getStdErr()); } // exit status delivery might get delayed diff --git a/utils/src/org/apache/cloudstack/utils/security/CertUtils.java b/utils/src/org/apache/cloudstack/utils/security/CertUtils.java index bf63d197d86..4915978d15b 100644 --- a/utils/src/org/apache/cloudstack/utils/security/CertUtils.java +++ b/utils/src/org/apache/cloudstack/utils/security/CertUtils.java @@ -40,6 +40,7 @@ import java.security.spec.InvalidKeySpecException; import java.security.spec.PKCS8EncodedKeySpec; import java.security.spec.X509EncodedKeySpec; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import javax.security.auth.x500.X500Principal; @@ -155,7 +156,7 @@ public class CertUtils { * @param signatureAlgorithm * @param validityDays * @param dnsNames - * @param publicIPAddresses + * @param ipAddresses * @return returns a X509Certificate * @throws IOException * @throws NoSuchAlgorithmException @@ -172,7 +173,7 @@ public class CertUtils { final String signatureAlgorithm, final int validityDays, final List dnsNames, - final List publicIPAddresses) throws IOException, NoSuchAlgorithmException, CertificateException, NoSuchProviderException, InvalidKeyException, SignatureException { + final List ipAddresses) throws IOException, NoSuchAlgorithmException, CertificateException, NoSuchProviderException, InvalidKeyException, SignatureException { final DateTime now = DateTime.now(DateTimeZone.UTC); final BigInteger serial = generateRandomBigInt(); final X500Principal subject = new X500Principal(subjectDN); @@ -199,8 +200,8 @@ public class CertUtils { new SubjectKeyIdentifierStructure(clientPublicKey)); final List subjectAlternativeNames = new ArrayList(); - if (publicIPAddresses != null) { - for (final String publicIPAddress: publicIPAddresses) { + if (ipAddresses != null) { + for (final String publicIPAddress: new HashSet<>(ipAddresses)) { if (Strings.isNullOrEmpty(publicIPAddress)) { continue; } @@ -208,7 +209,7 @@ public class CertUtils { } } if (dnsNames != null) { - for (final String dnsName : dnsNames) { + for (final String dnsName : new HashSet<>(dnsNames)) { if (Strings.isNullOrEmpty(dnsName)) { continue; } diff --git a/utils/src/org/apache/cloudstack/utils/security/KeyStoreUtils.java b/utils/src/org/apache/cloudstack/utils/security/KeyStoreUtils.java index 8690d39f24a..7d53aa60093 100644 --- a/utils/src/org/apache/cloudstack/utils/security/KeyStoreUtils.java +++ b/utils/src/org/apache/cloudstack/utils/security/KeyStoreUtils.java @@ -22,28 +22,35 @@ package org.apache.cloudstack.utils.security; import java.io.File; import java.io.IOException; +import com.cloud.utils.PropertiesUtil; import com.cloud.utils.script.Script; import com.google.common.base.Strings; public class KeyStoreUtils { + public static final String KS_SETUP_SCRIPT = "keystore-setup"; + public static final String KS_IMPORT_SCRIPT = "keystore-cert-import"; + public static final String KS_PASSPHRASE_PROPERTY = "keystore.passphrase"; - public static String defaultTmpKeyStoreFile = "/tmp/tmp.jks"; - public static String defaultKeystoreFile = "/cloud.jks"; - public static String defaultPrivateKeyFile = "/cloud.key"; - public static String defaultCsrFile = "/cloud.csr"; - public static String defaultCertFile = "/cloud.crt"; - public static String defaultCaCertFile = "/cloud.ca.crt"; - public static char[] defaultKeystorePassphrase = "vmops.com".toCharArray(); + public static final String KS_FILENAME = "/cloud.jks"; + public static final String KS_TMP_FILE = "/tmp/tmp.jks"; + public static final char[] DEFAULT_KS_PASSPHRASE = "vmops.com".toCharArray(); - public static String certNewlineEncoder = "^"; - public static String certSpaceEncoder = "~"; + public static final String CSR_FILENAME = "/cloud.csr"; + public static final String PKEY_FILENAME = "/cloud.key"; + public static final String CERT_FILENAME = "/cloud.crt"; + public static final String CACERT_FILENAME = "/cloud.ca.crt"; - public static String keyStoreSetupScript = "keystore-setup"; - public static String keyStoreImportScript = "keystore-cert-import"; - public static String passphrasePropertyName = "keystore.passphrase"; + public static final String CERT_NEWLINE_ENCODER = "^"; + public static final String CERT_SPACE_ENCODER = "~"; - public static String sshMode = "ssh"; - public static String agentMode = "agent"; + public static final String SSH_MODE = "ssh"; + public static final String AGENT_MODE = "agent"; + public static final String SECURED = "secured"; + + public static boolean isHostSecured() { + final File confFile = PropertiesUtil.findConfigFile("agent.properties"); + return confFile != null && confFile.exists() && new File(confFile.getParent() + CERT_FILENAME).exists(); + } public static void copyKeystore(final String keystorePath, final String tmpKeystorePath) throws IOException { if (Strings.isNullOrEmpty(keystorePath) || Strings.isNullOrEmpty(tmpKeystorePath)) {