From 3bf4e5c4982178e0d3e3d4a685d8f634cc1f1a23 Mon Sep 17 00:00:00 2001 From: Anurag Awasthi <43956255+anuragaw@users.noreply.github.com> Date: Thu, 30 May 2019 15:32:37 +0530 Subject: [PATCH 01/37] ui: configurable branding, keyboard list and hide-able columns through a new config.js file (#3258) We want to support hiding table columns, specifically in metrics table, through config file so that users can make the relevant bits hidden as per their organization. Current work will support the metrics table but can be extended to any table with minimal work in future. Config file will take the key of the metrics column from metrics.js file for the sake of minimal changes and simplicity of development. Problem: The keyboard list in the UI is not consistent across views such as in the instance wizard and in the register template form. There is also no way to custom about url/text and doc title and help URL in the UI. Root Cause: The list is hardcoded in the UI allowing no centralised configuration. Solution: Introduce a new config.js file installed at the /usr/share/cloudstackmanagement/webapp/config.js location. The config.js allows configurable keyboard list, about url/text, doc title, and help URL. Signed-off-by: Rohit Yadav --- ui/config.js | 40 ++++++++++++++++++++++++++++ ui/index.html | 11 +++++--- ui/l10n/en.js | 1 + ui/scripts/cloudStack.js | 2 +- ui/scripts/postLoad.js | 44 +++++++++++++++++++++++++++++++ ui/scripts/templates.js | 22 +++++----------- ui/scripts/ui/core.js | 13 ++++----- ui/scripts/ui/widgets/listView.js | 4 +++ 8 files changed, 108 insertions(+), 29 deletions(-) create mode 100644 ui/config.js create mode 100644 ui/scripts/postLoad.js diff --git a/ui/config.js b/ui/config.js new file mode 100644 index 00000000000..f715360fd0d --- /dev/null +++ b/ui/config.js @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Define custom options configurable by admins for UI +cloudStackOptions = { + aboutText: "label.app.name", // This is the text shown in the 'About' box + aboutTitle: "label.about.app", // This is the Application 'Title' shown in the 'About' box + docTitle: "label.app.name", // This is the Application 'Title' shown on browser tab. + + helpURL: "http://docs.cloudstack.apache.org/", // This is the URL that opens when users click Help + keyboardOptions: { + "us": "label.standard.us.keyboard", + "uk": "label.uk.keyboard", + "fr": "label.french.azerty.keyboard", + "jp": "label.japanese.keyboard", + "sc": "label.simplified.chinese.keyboard" + }, + hiddenFields: { + "metrics.zones":[], // Options - "name", "state", "clusters", "cpuused", "cpuallocated", "memused", "memallocated" + "metrics.clusters": [], // Options - "name", "state", "hosts", "cpuused", "cpuallocated", "memused", "memallocated" + "metrics.hosts": [], // Options - "name", "state", "powerstate", "instances", "cpuused", "memused", "network" + "metrics.storagepool": [], // Options - "name", "property", "disk", + "metrics.instances": [], // Options - "name", "state", "ipaddress", "zonename", "cpuused", "memused", "network", "disk" + "metrics.volumes": [] // Options - "name", "state", "vmname", "sizegb", "physicalsize", "utilization", "storagetype", "storage" + } +}; diff --git a/ui/index.html b/ui/index.html index e725b8153de..7556fbe9302 100644 --- a/ui/index.html +++ b/ui/index.html @@ -1762,6 +1762,7 @@ + @@ -1920,5 +1920,8 @@ + + + diff --git a/ui/l10n/en.js b/ui/l10n/en.js index 7e992e5a4a6..7a13a2c8a24 100644 --- a/ui/l10n/en.js +++ b/ui/l10n/en.js @@ -789,6 +789,7 @@ var dictionary = { "label.format":"Format", "label.format.lower":"format", "label.friday":"Friday", +"label.french.azerty.keyboard":"French AZERTY keyboard", "label.full":"Full", "label.full.path":"Full path", "label.gateway":"Gateway", diff --git a/ui/scripts/cloudStack.js b/ui/scripts/cloudStack.js index 7faa8c590ca..3d9ae472f5c 100644 --- a/ui/scripts/cloudStack.js +++ b/ui/scripts/cloudStack.js @@ -482,6 +482,6 @@ cloudStack.uiCustom.login(loginArgs); - document.title = _l('label.app.name'); + document.title = _l(cloudStackOptions.docTitle); }); })(cloudStack, jQuery); diff --git a/ui/scripts/postLoad.js b/ui/scripts/postLoad.js new file mode 100644 index 00000000000..9734bcb8de4 --- /dev/null +++ b/ui/scripts/postLoad.js @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Load this script after all scripts have executed to populate data +(function(cloudStack) { + + var loadListViewPreFilters = function(data, prefix) { + $.each(Object.keys(data), function(idx, key) { + if (key == "listView") { + // Load config flags + if (cloudStackOptions.hiddenFields[prefix]) { + var oldPreFilter = data.listView.preFilter; + data.listView.preFilter = function() { + // Hide config specified fields only for users. + var hiddenFields = isUser() ? cloudStackOptions.hiddenFields[prefix] : []; + if (oldPreFilter) { + return hiddenFields.concat(oldPreFilter()); + } + return hiddenFields; + } + } + } else if (data[key] && $.type(data[key]) == "object") { + loadListViewPreFilters(data[key], (prefix != null && prefix.length > 0) ? prefix + "." + key : key); + } + }); + } + + loadListViewPreFilters(cloudStack.sections, ""); + +})(cloudStack); diff --git a/ui/scripts/templates.js b/ui/scripts/templates.js index a051b2efefb..f3bed8b69a0 100755 --- a/ui/scripts/templates.js +++ b/ui/scripts/templates.js @@ -449,22 +449,12 @@ id: "", description: "" }); - items.push({ - id: "us", - description: "US Keboard" - }); - items.push({ - id: "uk", - description: "UK Keyboard" - }); - items.push({ - id: "jp", - description: "Japanese Keyboard" - }); - items.push({ - id: "sc", - description: "Simplified Chinese" - }); + for (var key in cloudStackOptions.keyboardOptions) { + items.push({ + id: key, + description: _l(cloudStackOptions.keyboardOptions[key]) + }); + } args.response.success({ data: items }); diff --git a/ui/scripts/ui/core.js b/ui/scripts/ui/core.js index 3f8bb445343..09a3ebcbc70 100644 --- a/ui/scripts/ui/core.js +++ b/ui/scripts/ui/core.js @@ -317,23 +317,20 @@ if (this == 'label.help') { $link.addClass('help').click(function() { - var helpURL = 'http://cloudstack.apache.org/'; - - window.open(helpURL, '_blank'); - + window.open(cloudStackOptions.helpURL, '_blank'); return false; }); } if (this == 'label.about') { $link.addClass('about').click(function() { - var $logo = $('
').addClass('logo').text(_l('label.app.name')), - $version = $('
').addClass('version').text(g_cloudstackversion), + var $logo = $('
').addClass('logo').text(_l(cloudStackOptions.aboutText)), + $version = $('
').addClass('version').text(_l(g_cloudstackversion)), $about = $('
').addClass('about').append($logo).append($version); - + var $aboutDialog = $about.dialog({ modal: true, width: 300, - title: _l('label.about.app'), + title: _l(cloudStackOptions.aboutTitle), closeOnEscape: false, dialogClass: 'dialog-about', buttons: { diff --git a/ui/scripts/ui/widgets/listView.js b/ui/scripts/ui/widgets/listView.js index 69a5c7a7f47..b5f4b3636dd 100644 --- a/ui/scripts/ui/widgets/listView.js +++ b/ui/scripts/ui/widgets/listView.js @@ -862,6 +862,8 @@ if (groupableColumns) { $tr.addClass('groupable-header-columns').addClass('groupable-header'); $.each(fields, function(key) { + if ($.inArray(key, hiddenFields) != -1) + return true; var field = this; if (field.columns) { var colspan = Object.keys(field.columns).length; @@ -1205,6 +1207,8 @@ var reducedFields = {}; var idx = 0; $.each(fields, function(key) { + if ($.inArray(key, hiddenFields) != -1) + return true; var field = this; if (field.columns) { $.each(field.columns, function(innerKey) { From 964430ab23656323112552e5cc4c1c9198bfc56a Mon Sep 17 00:00:00 2001 From: Andrija Panic <45762285+andrijapanicsb@users.noreply.github.com> Date: Thu, 30 May 2019 12:47:48 +0200 Subject: [PATCH 02/37] vmware: update vmware reservations settings description (#3358) We do NOT always reserve VMware CPU/RAM resources - only when "vmware.reserve.cpu" or "vmware.reserve.mem" setting is set to TRUE - AND we do so, irrelevant if overprovisioning is active or not. Verified for both system VMs and user VMs. --- .../src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java index 10c3feb2609..4777b738391 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java @@ -157,11 +157,11 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co } public static final ConfigKey VmwareReserveCpu = new ConfigKey(Boolean.class, "vmware.reserve.cpu", "Advanced", "false", - "Specify whether or not to reserve CPU when not overprovisioning, In case of cpu overprovisioning we will always reserve cpu.", true, ConfigKey.Scope.Cluster, + "Specify whether or not to reserve CPU when deploying an instance.", true, ConfigKey.Scope.Cluster, null); public static final ConfigKey VmwareReserveMemory = new ConfigKey(Boolean.class, "vmware.reserve.mem", "Advanced", "false", - "Specify whether or not to reserve memory when not overprovisioning, In case of memory overprovisioning we will always reserve memory.", true, + "Specify whether or not to reserve memory when deploying an instance.", true, ConfigKey.Scope.Cluster, null); protected ConfigKey VmwareEnableNestedVirtualization = new ConfigKey(Boolean.class, "vmware.nested.virtualization", "Advanced", "false", From fd4309cc3dc3c001a9cf77d89250af648158b5f0 Mon Sep 17 00:00:00 2001 From: Andrija Panic <45762285+andrijapanicsb@users.noreply.github.com> Date: Fri, 31 May 2019 06:02:59 +0200 Subject: [PATCH 03/37] server: increase POST timeout for local template upload (#3356) Increase template upload POST timeout for local template upload to 1 hr or 60 minutes. --- .../java/com/cloud/storage/ImageStoreUploadMonitorImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/storage/ImageStoreUploadMonitorImpl.java b/server/src/main/java/com/cloud/storage/ImageStoreUploadMonitorImpl.java index 10406b569db..b5ee6b58b38 100755 --- a/server/src/main/java/com/cloud/storage/ImageStoreUploadMonitorImpl.java +++ b/server/src/main/java/com/cloud/storage/ImageStoreUploadMonitorImpl.java @@ -110,7 +110,7 @@ public class ImageStoreUploadMonitorImpl extends ManagerBase implements ImageSto static final ConfigKey UploadMonitoringInterval = new ConfigKey("Advanced", Integer.class, "upload.monitoring.interval", "60", "Interval (in seconds) to check the status of volumes that are uploaded using HTTP POST request", true); - static final ConfigKey UploadOperationTimeout = new ConfigKey("Advanced", Integer.class, "upload.operation.timeout", "10", + static final ConfigKey UploadOperationTimeout = new ConfigKey("Advanced", Integer.class, "upload.operation.timeout", "60", "Time (in minutes) to wait before abandoning volume upload using HTTP POST request", true); @Override From 8c387f9de6d76cfa983b6fa7f39b2e9df4be4266 Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Fri, 31 May 2019 10:27:36 +0530 Subject: [PATCH 04/37] vmware: fix potential NPE when memory hotplug capability is checked (#3362) This fixes potential NPE case when memory hotpluggability is checked based on the guest OS descriptor. Signed-off-by: Rohit Yadav --- .../src/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vmware-base/src/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java b/vmware-base/src/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java index b700b6d8f63..1ab325be14c 100644 --- a/vmware-base/src/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java +++ b/vmware-base/src/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java @@ -3281,7 +3281,7 @@ public class VirtualMachineMO extends BaseMO { virtualHardwareVersion = getVirtualHardwareVersion(); // Check if guest operating system supports memory hotadd - if (guestOsDescriptor.isSupportsMemoryHotAdd()) { + if (guestOsDescriptor != null && guestOsDescriptor.isSupportsMemoryHotAdd()) { guestOsSupportsMemoryHotAdd = true; } // Check if virtual machine is using hardware version 7 or later. From bd780303853132c696cf8e2ab20d9e1641fd5a25 Mon Sep 17 00:00:00 2001 From: ustcweizhou Date: Fri, 31 May 2019 08:53:55 +0200 Subject: [PATCH 05/37] server: update dhcp configurations in vrs while update default nic of running vms (#3205) In virtual routers, there are different dnsmasq settings for default nic and non-default nic on vm. We need to update dhcp informations on network vrs when default nic is changed. For example, if 172.16.1.135 is non-default nic of vm VPC1-001-001, then root@r-22-VM:~# cat /etc/dhcphosts.txt 02:00:1d:15:00:05,set:172_16_1_135,172.16.1.135,VPC1-001-001,710h root@r-22-VM:~# cat /etc/dhcpopts.txt 172_16_1_135,3 172_16_1_135,6 172_16_1_135,15 If it is default nic,then root@r-22-VM:~# cat /etc/dhcpopts.txt root@r-22-VM:~# cat /etc/dhcphosts.txt 02:00:1d:15:00:05,172.16.1.135,VPC1-001-001,757h Fixes #3201 --- server/src/com/cloud/vm/UserVmManagerImpl.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/server/src/com/cloud/vm/UserVmManagerImpl.java b/server/src/com/cloud/vm/UserVmManagerImpl.java index 7ba282f4b18..8857fe91ee1 100644 --- a/server/src/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/com/cloud/vm/UserVmManagerImpl.java @@ -1448,6 +1448,19 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir newNetworkOfferingId, null, 0L, VirtualMachine.class.getName(), vmInstance.getUuid(), vmInstance.isDisplay()); UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NETWORK_OFFERING_ASSIGN, vmInstance.getAccountId(), vmInstance.getDataCenterId(), vmInstance.getId(), oldNicIdString, oldNetworkOfferingId, null, 0L, VirtualMachine.class.getName(), vmInstance.getUuid(), vmInstance.isDisplay()); + + if (vmInstance.getState() != State.Stopped) { + try { + VirtualMachineProfile vmProfile = new VirtualMachineProfileImpl(vmInstance); + User callerUser = _accountMgr.getActiveUser(CallContext.current().getCallingUserId()); + ReservationContext context = new ReservationContextImpl(null, null, callerUser, caller); + DeployDestination dest = new DeployDestination(dc, null, null, null); + _networkMgr.prepare(vmProfile, dest, context); + } catch (final Exception e) { + s_logger.info("Got exception: ", e); + } + } + return _vmDao.findById(vmInstance.getId()); } From b8522c97cb9204e16a9dd4c138bdbf0046992820 Mon Sep 17 00:00:00 2001 From: ustcweizhou Date: Fri, 31 May 2019 08:54:33 +0200 Subject: [PATCH 06/37] server: allow dedicate ip range to a domain if ips are used by an accout in the domain (#3206) when we dedicate public ip range to a domain but some ips are used by an account in the domain, the operation should be allowed but actually fails for now. It is because cloudstack check if ips are used by same account by account name, However, accountName is null when dedicate public ip range to a domain. Modify the code to check account id only when dedicate ip range to account. --- .../configuration/ConfigurationManagerImpl.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/server/src/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/com/cloud/configuration/ConfigurationManagerImpl.java index 9075ee903a5..d0230755d47 100755 --- a/server/src/com/cloud/configuration/ConfigurationManagerImpl.java +++ b/server/src/com/cloud/configuration/ConfigurationManagerImpl.java @@ -3601,9 +3601,9 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati } boolean isDomainSpecific = false; - List domainVln = _domainVlanMapDao.listDomainVlanMapsByVlan(vlanRange.getId()); + List domainVlan = _domainVlanMapDao.listDomainVlanMapsByVlan(vlanRange.getId()); // Check for domain wide pool. It will have an entry for domain_vlan_map. - if (domainVln != null && !domainVln.isEmpty()) { + if (domainVlan != null && !domainVlan.isEmpty()) { isDomainSpecific = true; } @@ -3760,10 +3760,10 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati forSystemVms = ip.isForSystemVms(); final Long allocatedToAccountId = ip.getAllocatedToAccountId(); if (allocatedToAccountId != null) { - final Account accountAllocatedTo = _accountMgr.getActiveAccountById(allocatedToAccountId); - if (!accountAllocatedTo.getAccountName().equalsIgnoreCase(accountName)) { + if (vlanOwner != null && allocatedToAccountId != vlanOwner.getId()) { throw new InvalidParameterValueException(ip.getAddress() + " Public IP address in range is allocated to another account "); } + final Account accountAllocatedTo = _accountMgr.getActiveAccountById(allocatedToAccountId); if (vlanOwner == null && domain != null && domain.getId() != accountAllocatedTo.getDomainId()){ throw new InvalidParameterValueException(ip.getAddress() + " Public IP address in range is allocated to another domain/account "); @@ -3824,9 +3824,9 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati } boolean isDomainSpecific = false; - final List domainVln = _domainVlanMapDao.listDomainVlanMapsByVlan(vlanDbId); + final List domainVlan = _domainVlanMapDao.listDomainVlanMapsByVlan(vlanDbId); // Check for domain wide pool. It will have an entry for domain_vlan_map. - if (domainVln != null && !domainVln.isEmpty()) { + if (domainVlan != null && !domainVlan.isEmpty()) { isDomainSpecific = true; } @@ -3879,7 +3879,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati // decrement resource count for dedicated public ip's _resourceLimitMgr.decrementResourceCount(acctVln.get(0).getAccountId(), ResourceType.public_ip, new Long(ips.size())); return true; - } else if (isDomainSpecific && _domainVlanMapDao.remove(domainVln.get(0).getId())) { + } else if (isDomainSpecific && _domainVlanMapDao.remove(domainVlan.get(0).getId())) { s_logger.debug("Remove the vlan from domain_vlan_map successfully."); return true; } else { From 2f268fbb52203cae96d0cff57c189600abab1390 Mon Sep 17 00:00:00 2001 From: Richard Lawley Date: Fri, 31 May 2019 08:05:42 +0100 Subject: [PATCH 07/37] systemvm: fix VR issues with Multiple Public Subnets (#3361) This PR resolves 2 issues related to Virtual Routers with multiple public interfaces, and works around a third. - Fixes #3353 - Adds missing throw routes for eth0/eth1 to eth3+ when there are >1 public IPs - Fixes #3168 - Incorrect marks set on some static NAT rules (some code references were changed from hex(int(interfacenum)) to hex(100 + int(interfacenum)) - this change just adds the remaining ones - Fixes #3352 - Work around that sends Gratuitous ARP messages when a HA VR becomes master to work around the problem of the MAC address being different between HA VRs. If that issue is fixed properly (i.e. a database entry for the subsequent interfaces so they can be static) then this is unnecessary, though should not cause any problems. --- systemvm/debian/opt/cloud/bin/configure.py | 4 +-- systemvm/debian/opt/cloud/bin/cs/CsAddress.py | 11 ++++---- .../debian/opt/cloud/bin/cs/CsRedundant.py | 27 +++++++++++++++++++ 3 files changed, 34 insertions(+), 8 deletions(-) diff --git a/systemvm/debian/opt/cloud/bin/configure.py b/systemvm/debian/opt/cloud/bin/configure.py index 253eb7c57fe..4df99111bfd 100755 --- a/systemvm/debian/opt/cloud/bin/configure.py +++ b/systemvm/debian/opt/cloud/bin/configure.py @@ -858,7 +858,7 @@ class CsForwardingRules(CsDataBag): rule['protocol'], rule['protocol'], public_fwports, - hex(int(public_fwinterface[3:])) + hex(100 + int(public_fwinterface[3:])) ) fw6 = "-A PREROUTING -d %s/32 -i %s -p %s -m %s --dport %s -m state --state NEW -j CONNMARK --save-mark --nfmask 0xffffffff --ctmask 0xffffffff" % \ ( @@ -927,7 +927,7 @@ class CsForwardingRules(CsDataBag): rule["internal_ip"]]) self.fw.append(["mangle", "", "-I PREROUTING -s %s/32 -m state --state NEW -j MARK --set-xmark %s/0xffffffff" % - (rule["internal_ip"], hex(int(device[len("eth"):])))]) + (rule["internal_ip"], hex(100 + int(device[len("eth"):])))]) self.fw.append(["nat", "front", "-A PREROUTING -d %s/32 -j DNAT --to-destination %s" % (rule["public_ip"], rule["internal_ip"])]) self.fw.append(["nat", "front", diff --git a/systemvm/debian/opt/cloud/bin/cs/CsAddress.py b/systemvm/debian/opt/cloud/bin/cs/CsAddress.py index ab0cee60039..8e678251fe3 100755 --- a/systemvm/debian/opt/cloud/bin/cs/CsAddress.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsAddress.py @@ -258,7 +258,7 @@ class CsIP: def __init__(self, dev, config): self.dev = dev - self.dnum = hex(int(dev[3:])) + self.dnum = hex(100 + int(dev[3:])) self.iplist = {} self.address = {} self.list() @@ -518,12 +518,11 @@ class CsIP: if method == "add": if not self.config.is_vpc(): - # treat the first IP on a interface as special case to set up the routing rules - if self.get_type() in ["public"] and (len(self.iplist) == 1): - CsHelper.execute("sudo ip route add throw " + self.config.address().dbag['eth0'][0]['network'] + " table " + tableName + " proto static") - CsHelper.execute("sudo ip route add throw " + self.config.address().dbag['eth1'][0]['network'] + " table " + tableName + " proto static") + if self.get_type() in ["public"]: + route.set_route("table %s throw %s proto static" % (tableName, self.config.address().dbag['eth0'][0]['network'])) + route.set_route("table %s throw %s proto static" % (tableName, self.config.address().dbag['eth1'][0]['network'])) - # add 'defaul via gateway' rule in the device specific routing table + # add 'default via gateway' rule in the device specific routing table if "gateway" in self.address and self.address["gateway"] and self.address["gateway"] != "None": route.add_route(self.dev, self.address["gateway"]) if "network" in self.address and self.address["network"]: diff --git a/systemvm/debian/opt/cloud/bin/cs/CsRedundant.py b/systemvm/debian/opt/cloud/bin/cs/CsRedundant.py index 3ade4a2a979..25a4a1a9438 100755 --- a/systemvm/debian/opt/cloud/bin/cs/CsRedundant.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsRedundant.py @@ -351,6 +351,33 @@ class CsRedundant(object): interfaces = [interface for interface in self.address.get_interfaces() if interface.is_public()] CsHelper.reconfigure_interfaces(self.cl, interfaces) + + public_devices = list(set([interface.get_device() for interface in interfaces])) + if len(public_devices) > 1: + # Handle specific failures when multiple public interfaces + + public_devices.sort() + + # Ensure the default route is added, or outgoing traffic from VMs with static NAT on + # the subsequent interfaces will go from he wrong IP + route = CsRoute() + dev = '' + for interface in interfaces: + if dev == interface.get_device(): + continue + dev = interface.get_device() + gateway = interface.get_gateway() + if gateway: + route.add_route(dev, gateway) + + # The first public interface has a static MAC address between VRs. Subsequent ones don't, + # so an ARP announcement is needed on failover + for device in public_devices[1:]: + logging.info("Sending garp messages for IPs on %s" % device) + for interface in interfaces: + if interface.get_device() == device: + CsHelper.execute("arping -I %s -U %s -c 1" % (device, interface.get_ip())) + logging.info("Router switched to master mode") def _collect_ignore_ips(self): From 6c6f40fd4d9874fc098ae41fcffb2be4a99f830d Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 3 Jun 2019 12:52:19 +0530 Subject: [PATCH 08/37] server: fix for public IP addresses filtering (#3368) Added missing parameter in listPublicIpAddresses API. Signed-off-by: Abhishek Kumar --- .../user/address/ListPublicIpAddressesCmd.java | 11 +++++++++++ .../com/cloud/server/ManagementServerImpl.java | 14 ++++++++++---- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/address/ListPublicIpAddressesCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/address/ListPublicIpAddressesCmd.java index d590081104a..d25d167636f 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/address/ListPublicIpAddressesCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/address/ListPublicIpAddressesCmd.java @@ -86,6 +86,13 @@ public class ListPublicIpAddressesCmd extends BaseListTaggedResourcesCmd { description = "lists all public IP addresses associated to the network specified") private Long associatedNetworkId; + @Parameter(name = ApiConstants.NETWORK_ID, + type = CommandType.UUID, + entityType = NetworkResponse.class, + description = "lists all public IP addresses by source network ID", + since = "4.13.0") + private Long networkId; + @Parameter(name = ApiConstants.IS_SOURCE_NAT, type = CommandType.BOOLEAN, description = "list only source NAT IP addresses") private Boolean isSourceNat; @@ -133,6 +140,10 @@ public class ListPublicIpAddressesCmd extends BaseListTaggedResourcesCmd { return associatedNetworkId; } + public Long getNetworkId() { + return networkId; + } + public Boolean isSourceNat() { return isSourceNat; } diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index b211cab27a2..af2c6c8003e 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -37,7 +37,6 @@ import javax.crypto.spec.SecretKeySpec; import javax.inject.Inject; import javax.naming.ConfigurationException; -import com.cloud.storage.ScopeType; import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.affinity.AffinityGroupProcessor; import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao; @@ -217,10 +216,10 @@ import org.apache.cloudstack.api.command.admin.usage.AddTrafficTypeCmd; import org.apache.cloudstack.api.command.admin.usage.DeleteTrafficMonitorCmd; import org.apache.cloudstack.api.command.admin.usage.DeleteTrafficTypeCmd; import org.apache.cloudstack.api.command.admin.usage.GenerateUsageRecordsCmd; -import org.apache.cloudstack.api.command.admin.usage.ListUsageRecordsCmd; import org.apache.cloudstack.api.command.admin.usage.ListTrafficMonitorsCmd; import org.apache.cloudstack.api.command.admin.usage.ListTrafficTypeImplementorsCmd; import org.apache.cloudstack.api.command.admin.usage.ListTrafficTypesCmd; +import org.apache.cloudstack.api.command.admin.usage.ListUsageRecordsCmd; import org.apache.cloudstack.api.command.admin.usage.ListUsageTypesCmd; import org.apache.cloudstack.api.command.admin.usage.RemoveRawUsageRecordsCmd; import org.apache.cloudstack.api.command.admin.usage.UpdateTrafficTypeCmd; @@ -634,6 +633,7 @@ import com.cloud.storage.GuestOSHypervisor; import com.cloud.storage.GuestOSHypervisorVO; import com.cloud.storage.GuestOSVO; import com.cloud.storage.GuestOsCategory; +import com.cloud.storage.ScopeType; import com.cloud.storage.StorageManager; import com.cloud.storage.StoragePool; import com.cloud.storage.Volume; @@ -1848,6 +1848,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe final Object keyword = cmd.getKeyword(); final Long physicalNetworkId = cmd.getPhysicalNetworkId(); final Long associatedNetworkId = cmd.getAssociatedNetworkId(); + final Long sourceNetworkId = cmd.getNetworkId(); final Long zone = cmd.getZoneId(); final String address = cmd.getIpAddress(); final Long vlan = cmd.getVlanId(); @@ -1893,7 +1894,8 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe sb.and("vlanDbId", sb.entity().getVlanId(), SearchCriteria.Op.EQ); sb.and("id", sb.entity().getId(), SearchCriteria.Op.EQ); sb.and("physicalNetworkId", sb.entity().getPhysicalNetworkId(), SearchCriteria.Op.EQ); - sb.and("associatedNetworkIdEq", sb.entity().getAssociatedWithNetworkId(), SearchCriteria.Op.EQ); + sb.and("associatedNetworkId", sb.entity().getAssociatedWithNetworkId(), SearchCriteria.Op.EQ); + sb.and("sourceNetworkId", sb.entity().getSourceNetworkId(), SearchCriteria.Op.EQ); sb.and("isSourceNat", sb.entity().isSourceNat(), SearchCriteria.Op.EQ); sb.and("isStaticNat", sb.entity().isOneToOneNat(), SearchCriteria.Op.EQ); sb.and("vpcId", sb.entity().getVpcId(), SearchCriteria.Op.EQ); @@ -1991,7 +1993,11 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe } if (associatedNetworkId != null) { - sc.setParameters("associatedNetworkIdEq", associatedNetworkId); + sc.setParameters("associatedNetworkId", associatedNetworkId); + } + + if (sourceNetworkId != null) { + sc.setParameters("sourceNetworkId", sourceNetworkId); } if (forDisplay != null) { From 6946f41784423788e71441255e67c2b9b1c3633d Mon Sep 17 00:00:00 2001 From: Sven Vogel Date: Mon, 3 Jun 2019 09:30:45 +0200 Subject: [PATCH 09/37] Fix template size for managed storage / refactor cloud-install-sys-tmplt and createtmplt.sh (#3346) * refactor cloud-install-sys-tmplt and createtmplt.sh * move qemu-img check to kvm / add more comments if qemu-img not available / set virtual size only for qcow2 * add comments to the failed and get options block / rename vars in failed block / typo Fix * add comments to the failed and get options block / rename vars in failed block / typo Fix to cloudtmplt.sh --- .../storage/secondary/cloud-install-sys-tmplt | 300 +++++++++--------- scripts/storage/secondary/createtmplt.sh | 184 +++++------ 2 files changed, 219 insertions(+), 265 deletions(-) diff --git a/scripts/storage/secondary/cloud-install-sys-tmplt b/scripts/storage/secondary/cloud-install-sys-tmplt index 91b3a7c9088..6bb8d20856b 100755 --- a/scripts/storage/secondary/cloud-install-sys-tmplt +++ b/scripts/storage/secondary/cloud-install-sys-tmplt @@ -1,5 +1,4 @@ #!/bin/bash -# $Id: installrtng.sh 11251 2010-07-23 23:40:44Z abhishek $ $HeadURL: svn://svn.lab.vmops.com/repos/vmdev/java/scripts/storage/secondary/installrtng.sh $ # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -20,19 +19,35 @@ usage() { - printf "Usage: %s: -m -f [-h ] [ -s ][-u ] [-F ] [-e