From d126cd21ea3e9aedb9031ce8331c59ccbfb8d467 Mon Sep 17 00:00:00 2001 From: Daan Hoogland Date: Wed, 13 Jun 2018 16:45:30 +0200 Subject: [PATCH 1/4] comply with api key constraint --- server/src/com/cloud/user/AccountManagerImpl.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/com/cloud/user/AccountManagerImpl.java b/server/src/com/cloud/user/AccountManagerImpl.java index dc9fdc0a9e5..9129976900d 100644 --- a/server/src/com/cloud/user/AccountManagerImpl.java +++ b/server/src/com/cloud/user/AccountManagerImpl.java @@ -1734,6 +1734,8 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M UserVO newUser = new UserVO(user); user.setExternalEntity(user.getUuid()); user.setUuid(UUID.randomUUID().toString()); + user.setApiKey(null); + user.setSecretKey(null); _userDao.update(user.getId(),user); newUser.setAccountId(newAccountId); boolean success = _userDao.remove(user.getId()); From c87ca1b26282bc818f997bec236a3766d041c8dd Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Tue, 19 Jun 2018 17:04:44 +0530 Subject: [PATCH 2/4] packaging: use libuuid x86_64 package for cloudstack-common (#2706) * packaging: use libuuid x86_64 package for cloudstack-common Signed-off-by: Rohit Yadav * 64 bit links is packaged * post scan filter to exclude libuuid.so.1 * Revert "packaging: use libuuid x86_64 package for cloudstack-common" This reverts commit b3fb8957fe4e98c85949be2010f0316c89d535a9. * post scan filter to exclude libuuid.so.1 (centos63) * revert removal of 32 bit support for vhd-util libs --- packaging/centos63/cloud.spec | 1 + packaging/centos7/cloud.spec | 1 + .../systemvmtemplate/scripts/install_systemvm_packages.sh | 6 ++++-- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/packaging/centos63/cloud.spec b/packaging/centos63/cloud.spec index 87d2c1ec578..68e2c56f5bf 100644 --- a/packaging/centos63/cloud.spec +++ b/packaging/centos63/cloud.spec @@ -109,6 +109,7 @@ Obsoletes: cloud-daemonize < 4.1.0 Group: System Environment/Libraries %description common The Apache CloudStack files shared between agent and management server +%global __requires_exclude ^libuuid\\.so\\.1$ %package agent Summary: CloudStack Agent for KVM hypervisors diff --git a/packaging/centos7/cloud.spec b/packaging/centos7/cloud.spec index 96b679c8955..a4f4dcc1b4c 100644 --- a/packaging/centos7/cloud.spec +++ b/packaging/centos7/cloud.spec @@ -93,6 +93,7 @@ Requires: python-netaddr Group: System Environment/Libraries %description common The Apache CloudStack files shared between agent and management server +%global __requires_exclude ^libuuid\\.so\\.1$ %package agent Summary: CloudStack Agent for KVM hypervisors diff --git a/tools/appliance/systemvmtemplate/scripts/install_systemvm_packages.sh b/tools/appliance/systemvmtemplate/scripts/install_systemvm_packages.sh index c1c3e94e0a8..585b38d4eb0 100644 --- a/tools/appliance/systemvmtemplate/scripts/install_systemvm_packages.sh +++ b/tools/appliance/systemvmtemplate/scripts/install_systemvm_packages.sh @@ -78,11 +78,13 @@ function install_packages() { apt-get clean apt-get autoclean - #32 bit architecture support:: not required for 32 bit template + ${apt_get} install links + + #32 bit architecture support for vhd-util: not required for 32 bit template if [ "${arch}" != "i386" ]; then dpkg --add-architecture i386 apt-get update - ${apt_get} install links:i386 libuuid1:i386 libc6:i386 + ${apt_get} install libuuid1:i386 libc6:i386 fi # Install xenserver guest utilities as debian repos don't have it From 08a59e89c397de0911cf2a22d010536eeeaa6a2f Mon Sep 17 00:00:00 2001 From: Slair1 Date: Tue, 19 Jun 2018 14:19:26 -0500 Subject: [PATCH 3/4] Source NAT option on Private Gateway (#2681) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Using Source NAT option on Private Gateway does not work This fixes #2680 ## Description When you use the Source NAT feature of Private Gateways on a VPC. This should Source NAT all traffic from CloudStack VMs going towards IPs reachable through Private Gateways. This change in this PR, stops adding the Source CIDR to SNAT rules. This should be discussed/reviewed, but i can see no reason why the Source CIDR is needed. There can only be one SNAT IP per interface, except for Static (one-to-one) NATs, which still work with this change in place. The outbound interface is what matters in the rule. ##### SUMMARY There is a bug in the Private Gateway functionality, when Source NAT is enabled for the Private Gateway. When the SNAT is added to iptables, it has the source CIDR of the private gateway subnet. Since no VMs live in that private gateway subnet, the SNAT doesn’t work. ##### STEPS TO REPRODUCE Below is an example: - VMs have IP addresses in the 10.0.0.0/24 subnet. - The Private Gateway address is 10.101.141.2/30 In the outputs below, the SOURCE field for the new SNAT (eth3) only matches if the source is 10.101.141.0/30. Since the VM has an IP address in 10.0.0.0/24, the VMs don’t get SNAT’d as they should when talking across the private gateway. The SOURCE should be set to ANYWHERE. ##### BEFORE ADDING PRIVATE GATEWAY ~~~ Chain POSTROUTING (policy ACCEPT 1 packets, 52 bytes) pkts bytes target prot opt in out source destination 2 736 SNAT all -- any eth2 10.0.0.0/24 anywhere to:10.0.0.1 16 1039 SNAT all -- any eth1 anywhere anywhere to:46.99.52.18 ~~~ ##### EXPECTED RESULTS ~~~ Chain POSTROUTING (policy ACCEPT 0 packets, 0 bytes) pkts bytes target prot opt in out source destination 0 0 SNAT all -- any eth3 anywhere anywhere to:10.101.141.2 2 736 SNAT all -- any eth2 anywhere anywhere to:10.0.0.1 23 1515 SNAT all -- any eth1 anywhere anywhere to:46.99.52.18 ~~~ ##### ACTUAL RESULTS ~~~ Chain POSTROUTING (policy ACCEPT 0 packets, 0 bytes) pkts bytes target prot opt in out source destination 0 0 SNAT all -- any eth3 10.101.141.0/30 anywhere to:10.101.141.2 2 736 SNAT all -- any eth2 10.0.0.0/24 anywhere to:10.0.0.1 23 1515 SNAT all -- any eth1 anywhere anywhere to:46.99.52.18 ~~~ ## Types of changes - [ ] Breaking change (fix or feature that would cause existing functionality to change) - [ ] New feature (non-breaking change which adds functionality) - [X] Bug fix (non-breaking change which fixes an issue) - [ ] Enhancement (improves an existing feature and functionality) - [ ] Cleanup (Code refactoring and cleanup, that may add test cases) ## GitHub Issue/PRs Fixes: #2680 ## Screenshots (if appropriate): ## How Has This Been Tested? ## Checklist: - [x] I have read the [CONTRIBUTING](https://github.com/apache/cloudstack/blob/master/CONTRIBUTING.md) document. - [x] My code follows the code style of this project. - [ ] My change requires a change to the documentation. - [ ] I have updated the documentation accordingly. Testing - [ ] I have added tests to cover my changes. - [ ] All relevant new and existing integration tests have passed. - [ ] A full integration testsuite with all test that can run on my environment has passed. --- systemvm/debian/opt/cloud/bin/configure.py | 9 +++------ systemvm/debian/opt/cloud/bin/cs/CsAddress.py | 7 ++++--- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/systemvm/debian/opt/cloud/bin/configure.py b/systemvm/debian/opt/cloud/bin/configure.py index e023a9fb94e..c8e3ff6e504 100755 --- a/systemvm/debian/opt/cloud/bin/configure.py +++ b/systemvm/debian/opt/cloud/bin/configure.py @@ -937,12 +937,9 @@ class CsForwardingRules(CsDataBag): self.fw.append(["filter", "", "-A FORWARD -i %s -o eth0 -d %s -m state --state NEW -j ACCEPT " % (device, rule["internal_ip"])]) - # Configure the hairpin nat - self.fw.append(["nat", "front", - "-A PREROUTING -d %s -i eth0 -j DNAT --to-destination %s" % (rule["public_ip"], rule["internal_ip"])]) - - self.fw.append(["nat", "front", "-A POSTROUTING -s %s -d %s -j SNAT -o eth0 --to-source %s" % - (self.getNetworkByIp(rule['internal_ip']), rule["internal_ip"], self.getGuestIp())]) + # Configure the hairpin snat + self.fw.append(["nat", "front", "-A POSTROUTING -s %s -d %s -j SNAT -o %s --to-source %s" % + (self.getNetworkByIp(rule['internal_ip']), rule["internal_ip"], self.getDeviceByIp(rule["internal_ip"]), self.getGuestIp())]) class IpTablesExecutor: diff --git a/systemvm/debian/opt/cloud/bin/cs/CsAddress.py b/systemvm/debian/opt/cloud/bin/cs/CsAddress.py index 3212dff71ee..10b6d3cfdbe 100755 --- a/systemvm/debian/opt/cloud/bin/cs/CsAddress.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsAddress.py @@ -476,9 +476,10 @@ class CsIP: self.fw.append(["", "front", "-A NETWORK_STATS_%s -o %s -s %s" % ("eth1", "eth1", guestNetworkCidr)]) - self.fw.append(["nat", "front", - "-A POSTROUTING -s %s -o %s -j SNAT --to-source %s" % - (guestNetworkCidr, self.dev, self.address['public_ip'])]) + if self.address["source_nat"]: + self.fw.append(["nat", "front", + "-A POSTROUTING -o %s -j SNAT --to-source %s" % + (self.dev, self.address['public_ip'])]) if self.get_type() in ["public"]: self.fw.append( From 39471c8c00d0e977872e9224e31de5b4dda9f051 Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Wed, 20 Jun 2018 12:25:16 +0530 Subject: [PATCH 4/4] configdrive: make fewer mountpoints on hosts (#2716) This ensure that fewer mount points are made on hosts for either primary storagepools or secondary storagepools. Signed-off-by: Rohit Yadav --- .../storage/configdrive/ConfigDrive.java | 14 +++++++++++--- .../storage/configdrive/ConfigDriveBuilder.java | 5 +++++ .../storage/configdrive/ConfigDriveTest.java | 2 +- .../LibvirtHandleConfigDriveCommandWrapper.java | 3 +-- .../network/element/ConfigDriveNetworkElement.java | 3 ++- .../resource/NfsSecondaryStorageResource.java | 7 ++++--- 6 files changed, 24 insertions(+), 10 deletions(-) diff --git a/engine/storage/configdrive/src/org/apache/cloudstack/storage/configdrive/ConfigDrive.java b/engine/storage/configdrive/src/org/apache/cloudstack/storage/configdrive/ConfigDrive.java index 07cfdc883ca..2fbc590c275 100644 --- a/engine/storage/configdrive/src/org/apache/cloudstack/storage/configdrive/ConfigDrive.java +++ b/engine/storage/configdrive/src/org/apache/cloudstack/storage/configdrive/ConfigDrive.java @@ -19,7 +19,6 @@ package org.apache.cloudstack.storage.configdrive; public class ConfigDrive { - public final static String CONFIGDRIVEFILENAME = "configdrive.iso"; public final static String CONFIGDRIVEDIR = "configdrive"; public static final String cloudStackConfigDriveName = "/cloudstack/"; @@ -27,11 +26,20 @@ public class ConfigDrive { /** * Creates the path to ISO file relative to mount point. - * The config driver path will have the following formated: {@link #CONFIGDRIVEDIR} + / + instanceName + / + {@link #CONFIGDRIVEFILENAME} + * The config driver path will have the following format: {@link #CONFIGDRIVEDIR} + / + instanceName + ".iso" * * @return config drive ISO file path */ public static String createConfigDrivePath(String instanceName) { - return ConfigDrive.CONFIGDRIVEDIR + "/" + instanceName + "/" + ConfigDrive.CONFIGDRIVEFILENAME; + return ConfigDrive.CONFIGDRIVEDIR + "/" + configIsoFileName(instanceName); + } + + /** + * Config Drive iso file name for an instance name + * @param instanceName + * @return + */ + public static String configIsoFileName(String instanceName) { + return instanceName + ".iso"; } } \ No newline at end of file diff --git a/engine/storage/configdrive/src/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilder.java b/engine/storage/configdrive/src/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilder.java index 55e7979a772..0675b43cb4a 100644 --- a/engine/storage/configdrive/src/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilder.java +++ b/engine/storage/configdrive/src/org/apache/cloudstack/storage/configdrive/ConfigDriveBuilder.java @@ -79,6 +79,11 @@ public class ConfigDriveBuilder { public static File base64StringToFile(String encodedIsoData, String folder, String fileName) throws IOException { byte[] decoded = Base64.decodeBase64(encodedIsoData.getBytes(StandardCharsets.US_ASCII)); Path destPath = Paths.get(folder, fileName); + try { + Files.createDirectories(destPath.getParent()); + } catch (final IOException e) { + LOG.warn("Exception hit while trying to recreate directory: " + destPath.getParent().toString()); + } return Files.write(destPath, decoded).toFile(); } diff --git a/engine/storage/configdrive/test/org/apache/cloudstack/storage/configdrive/ConfigDriveTest.java b/engine/storage/configdrive/test/org/apache/cloudstack/storage/configdrive/ConfigDriveTest.java index 81294d4331b..fd3d3db86a3 100644 --- a/engine/storage/configdrive/test/org/apache/cloudstack/storage/configdrive/ConfigDriveTest.java +++ b/engine/storage/configdrive/test/org/apache/cloudstack/storage/configdrive/ConfigDriveTest.java @@ -26,7 +26,7 @@ public class ConfigDriveTest { @Test public void testConfigDriveIsoPath() throws IOException { - Assert.assertEquals(ConfigDrive.createConfigDrivePath("i-x-y"), "configdrive/i-x-y/configdrive.iso"); + Assert.assertEquals(ConfigDrive.createConfigDrivePath("i-x-y"), "configdrive/i-x-y.iso"); } } \ No newline at end of file diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtHandleConfigDriveCommandWrapper.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtHandleConfigDriveCommandWrapper.java index 5e4ef485123..6baae85e221 100644 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtHandleConfigDriveCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtHandleConfigDriveCommandWrapper.java @@ -24,7 +24,6 @@ import java.nio.file.Path; import java.nio.file.Paths; import org.apache.cloudstack.storage.configdrive.ConfigDriveBuilder; -import org.apache.commons.io.FileUtils; import org.apache.log4j.Logger; import com.cloud.agent.api.Answer; @@ -67,7 +66,7 @@ public final class LibvirtHandleConfigDriveCommandWrapper extends CommandWrapper } } else { try { - FileUtils.deleteDirectory(isoPath.getParent().toFile()); + Files.deleteIfExists(isoPath); } catch (IOException e) { LOG.warn("Failed to delete config drive: " + isoPath.toAbsolutePath().toString()); return new Answer(command, false, "Failed due to exception: " + e.getMessage()); diff --git a/server/src/com/cloud/network/element/ConfigDriveNetworkElement.java b/server/src/com/cloud/network/element/ConfigDriveNetworkElement.java index a304ea43fa2..919d7bd9d18 100644 --- a/server/src/com/cloud/network/element/ConfigDriveNetworkElement.java +++ b/server/src/com/cloud/network/element/ConfigDriveNetworkElement.java @@ -375,8 +375,9 @@ public class ConfigDriveNetworkElement extends AdapterBase implements NetworkEle LOG.debug("Creating config drive ISO for vm: " + profile.getInstanceName()); + final String isoFileName = ConfigDrive.configIsoFileName(profile.getInstanceName()); final String isoPath = ConfigDrive.createConfigDrivePath(profile.getInstanceName()); - final String isoData = ConfigDriveBuilder.buildConfigDrive(profile.getVmData(), ConfigDrive.CONFIGDRIVEFILENAME, profile.getConfigDriveLabel()); + final String isoData = ConfigDriveBuilder.buildConfigDrive(profile.getVmData(), isoFileName, profile.getConfigDriveLabel()); final HandleConfigDriveIsoCommand configDriveIsoCommand = new HandleConfigDriveIsoCommand(isoPath, isoData, dataStore.getTO(), true); final Answer answer = agentManager.easySend(agentId, configDriveIsoCommand); diff --git a/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index 0bb923f8053..46fa1707a03 100644 --- a/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@ -42,6 +42,7 @@ import java.io.UnsupportedEncodingException; import java.net.InetAddress; import java.net.URI; import java.net.UnknownHostException; +import java.nio.file.Files; import java.nio.file.Path; import java.security.NoSuchAlgorithmException; import java.util.ArrayList; @@ -337,7 +338,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S Path tempDir = null; try { tempDir = java.nio.file.Files.createTempDirectory(ConfigDrive.CONFIGDRIVEDIR); - File tmpIsoFile = ConfigDriveBuilder.base64StringToFile(cmd.getIsoData(), tempDir.toAbsolutePath().toString(), ConfigDrive.CONFIGDRIVEFILENAME); + File tmpIsoFile = ConfigDriveBuilder.base64StringToFile(cmd.getIsoData(), tempDir.toAbsolutePath().toString(), cmd.getIsoFile()); copyLocalToNfs(tmpIsoFile, new File(cmd.getIsoFile()), cmd.getDestStore()); } catch (IOException | ConfigurationException e) { return new Answer(cmd, false, "Failed due to exception: " + e.getMessage()); @@ -355,11 +356,11 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S DataStoreTO dstore = cmd.getDestStore(); if (dstore instanceof NfsTO) { NfsTO nfs = (NfsTO) dstore; - String relativeTemplatePath = new File(cmd.getIsoFile()).getParent(); + String relativeTemplatePath = new File(cmd.getIsoFile()).getPath(); String nfsMountPoint = getRootDir(nfs.getUrl(), _nfsVersion); File tmpltPath = new File(nfsMountPoint, relativeTemplatePath); try { - FileUtils.deleteDirectory(tmpltPath); + Files.deleteIfExists(tmpltPath.toPath()); } catch (IOException e) { return new Answer(cmd, e); }