From 3506789b0b4d0da55f9713f19c1e0f41c16830ee Mon Sep 17 00:00:00 2001 From: Min Chen Date: Wed, 17 Dec 2014 22:33:24 -0800 Subject: [PATCH 001/391] CLOUDSTACK-8093:Not able to list shared templates by passing id. --- server/src/com/cloud/api/query/QueryManagerImpl.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/src/com/cloud/api/query/QueryManagerImpl.java b/server/src/com/cloud/api/query/QueryManagerImpl.java index 31273ec317d..a5e7d5c5612 100644 --- a/server/src/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/com/cloud/api/query/QueryManagerImpl.java @@ -3015,8 +3015,7 @@ public class QueryManagerImpl extends ManagerBase implements QueryService { // if template is not public, perform permission check here if (!template.isPublicTemplate() && caller.getType() != Account.ACCOUNT_TYPE_ADMIN) { - Account owner = _accountMgr.getAccount(template.getAccountId()); - _accountMgr.checkAccess(caller, null, true, owner); + _accountMgr.checkAccess(caller, null, false, template); } // if templateId is specified, then we will just use the id to From 27295d235d0db1eccb571b4f8ea010e9f1cf3c45 Mon Sep 17 00:00:00 2001 From: Gaurav Aradhye Date: Thu, 18 Dec 2014 15:23:13 +0530 Subject: [PATCH 002/391] CLOUDSTACK-8089: Fixed test_explicit_dedication.py test case and moved to maint folder for it is to be run separately Signed-off-by: pdion891 --- .../{ => maint}/test_explicit_dedication.py | 192 +++++++++--------- 1 file changed, 100 insertions(+), 92 deletions(-) rename test/integration/component/{ => maint}/test_explicit_dedication.py (53%) diff --git a/test/integration/component/test_explicit_dedication.py b/test/integration/component/maint/test_explicit_dedication.py similarity index 53% rename from test/integration/component/test_explicit_dedication.py rename to test/integration/component/maint/test_explicit_dedication.py index 71c4141100d..970795a1f7e 100644 --- a/test/integration/component/test_explicit_dedication.py +++ b/test/integration/component/maint/test_explicit_dedication.py @@ -16,82 +16,89 @@ # under the License. """ P1 tests for Storage motion """ -#Import Local Modules -import marvin -from marvin.cloudstackTestCase import * -from marvin.cloudstackAPI import * -from marvin.sshClient import SshClient -from marvin.lib.utils import * -from marvin.lib.base import * -from marvin.lib.common import * +# Import Local Modules +from marvin.cloudstackTestCase import cloudstackTestCase +from marvin.lib.utils import cleanup_resources +from marvin.cloudstackAPI import dedicateHost, releaseDedicatedHost +from marvin.lib.base import (Account, + ServiceOffering, + VirtualMachine, + AffinityGroup) +from marvin.lib.common import (get_domain, + get_zone, + get_template, + list_hosts, + list_virtual_machines) from nose.plugins.attrib import attr -#Import System modules -import time _multiprocess_shared_ = True + + class Services: + """Test explicit dedication """ def __init__(self): self.services = { - "disk_offering":{ - "displaytext": "Small", - "name": "Small", - "disksize": 1 - }, - "account": { - "email": "test@test.com", - "firstname": "Test", - "lastname": "User", - "username": "testexplicit", - # Random characters are appended in create account to - # ensure unique username generated each time - "password": "password", - }, - "virtual_machine" : + "disk_offering": { + "displaytext": "Small", + "name": "Small", + "disksize": 1 + }, + "account": { + "email": "test@test.com", + "firstname": "Test", + "lastname": "User", + "username": "testexplicit", + # Random characters are appended in create account to + # ensure unique username generated each time + "password": "password", + }, + "virtual_machine": { "affinity": { "name": "explicit", "type": "ExplicitDedication", }, - "hypervisor" : "XenServer", - }, - "small": + "hypervisor": "XenServer", + }, + "small": # Create a small virtual machine instance with disk offering { "displayname": "testserver", - "username": "root", # VM creds for SSH + "username": "root", # VM creds for SSH "password": "password", "ssh_port": 22, "hypervisor": 'XenServer', "privateport": 22, "publicport": 22, "protocol": 'TCP', - }, - "service_offerings": + }, + "service_offerings": { - "for-explicit": + "for-explicit": { - # Small service offering ID to for change VM - # service offering from medium to small + # Small service offering ID to for change VM + # service offering from medium to small "name": "For explicit", "displaytext": "For explicit", "cpunumber": 1, "cpuspeed": 500, "memory": 512 } - }, - "template": { + }, + "template": { "displaytext": "Cent OS Template", "name": "Cent OS Template", "passwordenabled": True, - }, - "sleep": 60, + }, + "sleep": 60, "timeout": 10, "ostype": 'CentOS 5.3 (64-bit)' } + class TestExplicitDedication(cloudstackTestCase): @classmethod @@ -106,37 +113,36 @@ class TestExplicitDedication(cloudstackTestCase): cls.services['mode'] = cls.zone.networktype cls.template = get_template( - cls.api_client, - cls.zone.id, - cls.services["ostype"] - ) + cls.api_client, + cls.zone.id, + cls.services["ostype"] + ) # Set Zones and disk offerings cls.services["small"]["zoneid"] = cls.zone.id cls.services["small"]["template"] = cls.template.id # Create VMs, NAT Rules etc cls.account = Account.create( - cls.api_client, - cls.services["account"], - domainid=cls.domain.id - ) + cls.api_client, + cls.services["account"], + domainid=cls.domain.id + ) cls.small_offering = ServiceOffering.create( - cls.api_client, - cls.services["service_offerings"]["for-explicit"] - ) - - #cls.ag = AffinityGroup.create(cls.api_client, cls.services["virtual_machine"]["affinity"], - # account=cls.services["account"], domainid=cls.domain.id) + cls.api_client, + cls.services["service_offerings"]["for-explicit"] + ) cls._cleanup = [ - cls.small_offering, - cls.account - ] + cls.small_offering, + cls.account + ] @classmethod def tearDownClass(cls): - cls.api_client = super(TestExplicitDedication, cls).getClsTestClient().getApiClient() + cls.api_client = super( + TestExplicitDedication, + cls).getClsTestClient().getApiClient() cleanup_resources(cls.api_client, cls._cleanup) return @@ -146,57 +152,58 @@ class TestExplicitDedication(cloudstackTestCase): self.cleanup = [] def tearDown(self): - #Clean up, terminate the created ISOs + # Clean up, terminate the created ISOs cleanup_resources(self.apiclient, self.cleanup) return - # This test requires multi host and at least one host which is empty (no vms should - # be running on that host). It explicitly dedicates empty host to an account, deploys - # a vm for that account and verifies that the vm gets deployed to the dedicated host. - @attr(tags=["advanced", "basic", "multihosts", "explicitdedication"], required_hardware="false") + # This test requires multi host and at least one host which is empty + # (no vms should be running on that host). It explicitly dedicates + # empty host to an account, deploys a vm for that account and verifies + # that the vm gets deployed to the dedicated host. + @attr( + tags=[ + "advanced", + "basic", + "multihosts", + "explicitdedication"], + required_hardware="false") def test_01_deploy_vm_with_explicit_dedication(self): """Test explicit dedication is placing vms of an account on dedicated hosts. """ # Validate the following # 1. Find and dedicate an empty host to an account. - # 2. Create an affinity group for explicit dedication. - # 3. Create a vm deployment by passing the affinity group as a parameter. - # 4. Validate the vm got deployed on the dedicated host. - # 5. Cleanup. + # 2. Create a vm deployment by passing the affinity group as a + # parameter. + # 3. Validate the vm got deployed on the dedicated host. + # 4. Cleanup. # list and find an empty hosts all_hosts = list_hosts( - self.apiclient, - type='Routing', - ) + self.apiclient, + type='Routing', + ) empty_host = None for host in all_hosts: vms_on_host = list_virtual_machines( - self.api_client, - hostid=host.id) + self.api_client, + hostid=host.id) if not vms_on_host: empty_host = host break - #If no empty host is found, return - if empty_host: - self.skipTest("Did not find any empty hosts, Skipping") - - # Create an affinity group for explicit dedication. - agCmd = createAffinityGroup.createAffinityGroupCmd() - agCmd.name = "explicit-affinity" - agCmd.displayText = "explicit-affinity" - agCmd.account = self.account.name - agCmd.domainid = self.account.domainid - agCmd.type = self.services['virtual_machine']['affinity']['type'] - self.apiclient.createAffinityGroup(agCmd) + # If no empty host is found, return + if empty_host is None: + self.skipTest("Did not find any empty hosts, Skipping") # dedicate the empty host to this account. dedicateCmd = dedicateHost.dedicateHostCmd() dedicateCmd.hostid = empty_host.id dedicateCmd.domainid = self.domain.id - self.apiclient.dedicateHost(dedicateCmd) + self.api_client.dedicateHost(dedicateCmd) + affinitygroup = AffinityGroup.list( + self.apiclient, + type="ExplicitDedication") # deploy vm on the dedicated resource. vm = VirtualMachine.create( @@ -205,22 +212,22 @@ class TestExplicitDedication(cloudstackTestCase): accountid=self.account.name, domainid=self.account.domainid, serviceofferingid=self.small_offering.id, - affinitygroupnames=["explicit-affinity"], + affinitygroupids=affinitygroup[0].id, mode=self.services["mode"] ) list_vm_response = list_virtual_machines( - self.apiclient, - id=vm.id - ) + self.apiclient, + id=vm.id + ) vm_response = list_vm_response[0] self.assertEqual( - vm_response.hostid, - empty_host.id, - "Check destination hostID of deployed VM" - ) + vm_response.hostid, + empty_host.id, + "Check destination hostID of deployed VM" + ) # release the dedicated host to this account. releaseCmd = releaseDedicatedHost.releaseDedicatedHostCmd() @@ -228,6 +235,7 @@ class TestExplicitDedication(cloudstackTestCase): releaseCmd.domainid = self.domain.id self.apiclient.releaseDedicatedHost(releaseCmd) - #Deletion of the created VM and affinity group is taken care as part of account clean + # Deletion of the created VM and affinity group is taken care as part + # of account clean return From 65c742cd66e032c91a6e12e9eae4343c7f18965f Mon Sep 17 00:00:00 2001 From: Jessica Wang Date: Thu, 18 Dec 2014 18:13:14 -0800 Subject: [PATCH 003/391] CLOUDSTACK-6744 > UI > zone wizard > baremetal hypervisor > support EIP ELB feature. --- ui/scripts/zoneWizard.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/scripts/zoneWizard.js b/ui/scripts/zoneWizard.js index 4ce4efbf9be..66b34699eb0 100755 --- a/ui/scripts/zoneWizard.js +++ b/ui/scripts/zoneWizard.js @@ -577,7 +577,7 @@ }); if (thisNetworkOffering.havingEIP == true && thisNetworkOffering.havingELB == true) { //EIP ELB - if (args.hypervisor == "VMware" || args.hypervisor == "BareMetal") { //VMware, BareMetal don't support EIP ELB + if (args.hypervisor == "VMware") { //VMware does not support EIP ELB return true; //move to next item in $.each() loop } if (args.context.zones[0]["network-model"] == "Advanced" && args.context.zones[0]["zone-advanced-sg-enabled"] == "on") { // Advanced SG-enabled zone doesn't support EIP ELB From d88126988b04bb8a58fb70e195722b01b45f05dc Mon Sep 17 00:00:00 2001 From: Gaurav Aradhye Date: Thu, 18 Dec 2014 15:58:36 +0530 Subject: [PATCH 004/391] CLOUDSTACK-8090: Moving test_dedicated_guest_vlan_ranges.py to maint folder for the test cases need to be run separately, serially Signed-off-by: SrikanteswaraRao Talluri --- .../component/{ => maint}/test_dedicate_guest_vlan_ranges.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/integration/component/{ => maint}/test_dedicate_guest_vlan_ranges.py (100%) diff --git a/test/integration/component/test_dedicate_guest_vlan_ranges.py b/test/integration/component/maint/test_dedicate_guest_vlan_ranges.py similarity index 100% rename from test/integration/component/test_dedicate_guest_vlan_ranges.py rename to test/integration/component/maint/test_dedicate_guest_vlan_ranges.py From f2c7ead8ee68de963a97eae9dc78bf524565691e Mon Sep 17 00:00:00 2001 From: Chandan Purushothama Date: Wed, 17 Dec 2014 18:22:54 -0800 Subject: [PATCH 005/391] CLOUDSTACK-8086: Simulator needs a Portable IP Range to execute Portable IP Test Cases Signed-off-by: SrikanteswaraRao Talluri --- tools/marvin/marvin/config/test_data.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/marvin/marvin/config/test_data.py b/tools/marvin/marvin/config/test_data.py index 5a468fe9a8a..4ab7e56e825 100644 --- a/tools/marvin/marvin/config/test_data.py +++ b/tools/marvin/marvin/config/test_data.py @@ -1434,11 +1434,11 @@ test_data = { "configurableData": { "portableIpRange": { - "gateway": "", - "netmask": "", - "startip": "", - "endip": "", - "vlan": "" + "gateway": "10.223.59.1", + "netmask": "255.255.255.0", + "startip": "10.223.59.200", + "endip": "10.223.59.240", + "vlan": "1000" }, "netscaler": { "ipaddress": "", From 0db63d87aa40de7493a6bc66b7686cf8922bc742 Mon Sep 17 00:00:00 2001 From: Gaurav Aradhye Date: Wed, 17 Dec 2014 17:35:35 +0530 Subject: [PATCH 006/391] CLOUDSTACK-8084: Fixed test_17_add_nic_different_zone in test_add_remove_network.py Signed-off-by: SrikanteswaraRao Talluri --- test/integration/component/test_add_remove_network.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/integration/component/test_add_remove_network.py b/test/integration/component/test_add_remove_network.py index aacd44b029a..2f685b54caa 100644 --- a/test/integration/component/test_add_remove_network.py +++ b/test/integration/component/test_add_remove_network.py @@ -1263,6 +1263,7 @@ class TestFailureScenariosAddNetworkToVM(cloudstackTestCase): self.debug("Creating isolated network in zone %s which is foreign to VM" % foreignZoneId) isolated_network = Network.create(self.apiclient,self.services["isolated_network"], + self.account.name, self.account.domainid, networkofferingid=self.isolated_network_offering.id) self.debug("Created isolated network %s in zone %s" % (isolated_network.id, foreignZoneId)) @@ -1274,7 +1275,7 @@ class TestFailureScenariosAddNetworkToVM(cloudstackTestCase): cmd.networkid = isolated_network.id with self.assertRaises(Exception) as e: - time.sleep(5) + time.sleep(5) self.apiclient.addNicToVirtualMachine(cmd) self.debug("addNicToVirtualMachine API failed with exception: %s" % e.exception) From 6c722c9d219ae3bedad12a28105800b096eeb147 Mon Sep 17 00:00:00 2001 From: Ashutosh K Date: Thu, 18 Dec 2014 11:58:56 +0530 Subject: [PATCH 007/391] CLOUDSTACK-8087: Fixed test_vpc_on_host_maintenance.py Signed-off-by: SrikanteswaraRao Talluri --- .../maint/test_vpc_on_host_maintenance.py | 161 ++++++++---------- 1 file changed, 73 insertions(+), 88 deletions(-) diff --git a/test/integration/component/maint/test_vpc_on_host_maintenance.py b/test/integration/component/maint/test_vpc_on_host_maintenance.py index eb3360a458f..8ee50bf3056 100644 --- a/test/integration/component/maint/test_vpc_on_host_maintenance.py +++ b/test/integration/component/maint/test_vpc_on_host_maintenance.py @@ -16,62 +16,64 @@ # under the License. from nose.plugins.attrib import attr -from marvin.cloudstackTestCase import * -from marvin.cloudstackAPI import * -from marvin.lib.utils import * -from marvin.lib.base import * -from marvin.lib.common import * +from marvin.cloudstackTestCase import cloudstackTestCase, unittest +from marvin.lib.utils import cleanup_resources +from marvin.lib.base import (Account, + Host, + VPC, + VpcOffering) +from marvin.lib.common import (get_domain, + get_zone, + get_template, + list_configurations) +import time class Services: + """Test VPC services """ def __init__(self): self.services = { - "account": { - "email": "test@test.com", - "firstname": "Test", - "lastname": "User", - "username": "test", - # Random characters are appended for unique - # username - "password": "password", - }, - "service_offering": { - "name": "Tiny Instance", - "displaytext": "Tiny Instance", - "cpunumber": 1, - "cpuspeed": 100, - "memory": 128, - }, - "vpc_offering": { - "name": 'VPC off', - "displaytext": 'VPC off', - "supportedservices": 'Dhcp,Dns,SourceNat,PortForwarding,Vpn,Lb,UserData,StaticNat,NetworkACL', - }, - "vpc": { - "name": "TestVPC", - "displaytext": "TestVPC", - "cidr": '10.0.0.1/24' - }, - "virtual_machine": { - "displayname": "Test VM", - "username": "root", - "password": "password", - "ssh_port": 22, - "hypervisor": 'XenServer', - # Hypervisor type should be same as - # hypervisor type of cluster - "privateport": 22, - "publicport": 22, - "protocol": 'TCP', - }, - "ostype": 'CentOS 5.3 (64-bit)', - # Cent OS 5.3 (64 bit) - "sleep": 60, - "timeout": 10 - } + "account": { + "email": "test@test.com", + "firstname": "Test", + "lastname": "User", + "username": "test", + # Random characters are appended for unique + # username + "password": "password", + }, + "vpc_offering": { + "name": 'VPC off', + "displaytext": 'VPC off', + "supportedservices": 'Dhcp,Dns,SourceNat,PortForwarding,Vpn,Lb,\ +UserData,StaticNat,NetworkACL', + }, + "vpc": { + "name": "TestVPC", + "displaytext": "TestVPC", + "cidr": '10.0.0.1/24' + }, + "virtual_machine": { + "displayname": "Test VM", + "username": "root", + "password": "password", + "ssh_port": 22, + "hypervisor": 'XenServer', + # Hypervisor type should be same as + # hypervisor type of cluster + "privateport": 22, + "publicport": 22, + "protocol": 'TCP', + }, + "ostype": 'CentOS 5.3 (64-bit)', + # Cent OS 5.3 (64 bit) + "sleep": 60, + "timeout": 10 + } + class TestVPCHostMaintenance(cloudstackTestCase): @@ -93,24 +95,20 @@ class TestVPCHostMaintenance(cloudstackTestCase): cls.services["virtual_machine"]["template"] = cls.template.id cls.services["mode"] = cls.zone.networktype - cls.service_offering = ServiceOffering.create( - cls.api_client, - cls.services["service_offering"] - ) cls.vpc_off = VpcOffering.create( cls.api_client, cls.services["vpc_offering"] ) cls.vpc_off.update(cls.api_client, state='Enabled') - hosts = Host.list( + cls.hosts = Host.list( cls.api_client, zoneid=cls.zone.id, listall=True, type='Routing' ) - if isinstance(hosts, list): - for host in hosts: + if isinstance(cls.hosts, list): + for host in cls.hosts: Host.enableMaintenance( cls.api_client, id=host.id @@ -124,7 +122,8 @@ class TestVPCHostMaintenance(cloudstackTestCase): id=host.id, listall=True ) - if hosts_states[0].resourcestate == 'PrepareForMaintenance': + if hosts_states[ + 0].resourcestate == 'PrepareForMaintenance': # Wait for sometimetill host goes in maintenance state time.sleep(cls.services["sleep"]) elif hosts_states[0].resourcestate == 'Maintenance': @@ -132,11 +131,11 @@ class TestVPCHostMaintenance(cloudstackTestCase): break elif timeout == 0: raise unittest.SkipTest( - "Failed to enable maintenance mode on %s" % host.name) + "Failed to enable maintenance mode on %s" % + host.name) timeout = timeout - 1 cls._cleanup = [ - cls.service_offering, cls.vpc_off ] return @@ -144,20 +143,13 @@ class TestVPCHostMaintenance(cloudstackTestCase): @classmethod def tearDownClass(cls): try: - #Cleanup resources used + # Cleanup resources used cleanup_resources(cls.api_client, cls._cleanup) - hosts = Host.list( - cls.api_client, - zoneid=cls.zone.id, - listall=True, - type='Routing' - ) - if isinstance(hosts, list): - for host in hosts: - Host.cancelMaintenance( - cls.api_client, - id=host.id - ) + for host in cls.hosts: + Host.cancelMaintenance( + cls.api_client, + id=host.id + ) hosts_states = Host.list( cls.api_client, id=host.id, @@ -165,7 +157,8 @@ class TestVPCHostMaintenance(cloudstackTestCase): ) if hosts_states[0].resourcestate != 'Enabled': raise Exception( - "Failed to cancel maintenance mode on %s" % (host.name)) + "Failed to cancel maintenance mode on %s" % + (host.name)) except Exception as e: raise Exception("Warning: Exception during cleanup : %s" % e) return @@ -184,18 +177,8 @@ class TestVPCHostMaintenance(cloudstackTestCase): def tearDown(self): try: - #Clean up, terminate the created network offerings + # Clean up, terminate the created network offerings cleanup_resources(self.apiclient, self.cleanup) - interval = list_configurations( - self.apiclient, - name='network.gc.interval' - ) - wait = list_configurations( - self.apiclient, - name='network.gc.wait' - ) - # Sleep to ensure that all resources are deleted - time.sleep(int(interval[0].value) + int(wait[0].value)) except Exception as e: raise Exception("Warning: Exception during cleanup : %s" % e) return @@ -243,7 +226,7 @@ class TestVPCHostMaintenance(cloudstackTestCase): ) if state: self.assertEqual( - vpc_networks[0].state, + vpc_networks[0].state.lower(), state, "VPC state should be '%s'" % state ) @@ -269,9 +252,10 @@ class TestVPCHostMaintenance(cloudstackTestCase): vpcofferingid=self.vpc_off.id, zoneid=self.zone.id, account=self.account.name, - domainid=self.account.domainid + domainid=self.account.domainid, + start=False ) - self.validate_vpc_network(vpc, state='Disabled') + self.validate_vpc_network(vpc, state='inactive') return @attr(tags=["advanced", "intervlan"]) @@ -295,9 +279,10 @@ class TestVPCHostMaintenance(cloudstackTestCase): vpcofferingid=self.vpc_off.id, zoneid=self.zone.id, account=self.account.name, - domainid=self.account.domainid + domainid=self.account.domainid, + start=False ) - self.validate_vpc_network(vpc, state='Disabled') + self.validate_vpc_network(vpc, state='inactive') interval = list_configurations( self.apiclient, name='network.gc.interval' From a771c7d496f8556c32fb025e8fd2ba5477d4bd97 Mon Sep 17 00:00:00 2001 From: Jessica Wang Date: Fri, 19 Dec 2014 14:25:32 -0800 Subject: [PATCH 008/391] CLOUDSTACK-6666: UI > network > VPC > Router > Public IP Addresses > IP Address detailView > Configuration tab > Load Balancing > Select VM screen > implement keyword search. --- ui/scripts/network.js | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/ui/scripts/network.js b/ui/scripts/network.js index 156b738f908..2db8d271896 100755 --- a/ui/scripts/network.js +++ b/ui/scripts/network.js @@ -3347,18 +3347,18 @@ dataProvider: function(args) { var itemData = $.isArray(args.context.multiRule) && args.context.subItemData ? args.context.subItemData : []; + var data = {}; + listViewDataProvider(args, data); + var networkid; - if ('vpc' in args.context) + if ('vpc' in args.context) { networkid = args.context.multiData.tier; - else + } else { networkid = args.context.ipAddresses[0].associatednetworkid; - - var data = { - page: args.page, - pageSize: pageSize, - networkid: networkid, - listAll: true - }; + } + $.extend(data, { + networkid: networkid + }); if (!args.context.projects) { $.extend(data, { From 48e996f43eb23aa75ce3ca75de02e1790e2fbe16 Mon Sep 17 00:00:00 2001 From: Jessica Wang Date: Fri, 19 Dec 2014 14:26:42 -0800 Subject: [PATCH 009/391] CLOUDSTACK-6666: UI > network > VPC > Router > Public IP Addresses > IP Address detailView > Configuration tab > Port Forwarding > Select VM screen > implement keyword search. --- ui/scripts/network.js | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/ui/scripts/network.js b/ui/scripts/network.js index 2db8d271896..cc843019bd5 100755 --- a/ui/scripts/network.js +++ b/ui/scripts/network.js @@ -4110,18 +4110,18 @@ dataProvider: singleVmSecondaryIPSubselect }, dataProvider: function(args) { - var networkid; - if ('vpc' in args.context) + var data = {}; + listViewDataProvider(args, data); + + var networkid; + if ('vpc' in args.context) { networkid = args.context.multiData.tier; - else + } else { networkid = args.context.ipAddresses[0].associatednetworkid; - - var data = { - page: args.page, - pageSize: pageSize, - listAll: true, - networkid: networkid - }; + } + $.extend(data, { + networkid: networkid + }); if (!args.context.projects) { $.extend(data, { From a29a1e0600a31476f219769e90bb582f243ef32e Mon Sep 17 00:00:00 2001 From: Jessica Wang Date: Fri, 19 Dec 2014 15:41:42 -0800 Subject: [PATCH 010/391] CLOUDSTACK-8102: UI > Quick Install Wizard > update admin > should encode parameter value once instead of twice before sending it to API. --- ui/scripts/installWizard.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/scripts/installWizard.js b/ui/scripts/installWizard.js index 85eaff00c3f..42f9a8031df 100644 --- a/ui/scripts/installWizard.js +++ b/ui/scripts/installWizard.js @@ -35,7 +35,7 @@ url: createURL('updateUser'), data: { id: cloudStack.context.users[0].userid, - password: md5Hashed ? $.md5(args.data.password) : todb(args.data.password) + password: md5Hashed ? $.md5(args.data.password) : args.data.password }, dataType: 'json', async: true, From c5d3a73c67056048c9640f16730bc5929627d486 Mon Sep 17 00:00:00 2001 From: Min Chen Date: Fri, 19 Dec 2014 16:43:02 -0800 Subject: [PATCH 011/391] CLOUDSTACK-8101: volume sync not working as expected - MS restart during upload volume leaves volume in hung state. --- .../storage/volume/VolumeServiceImpl.java | 15 +++++++++++---- .../storage/download/DownloadMonitorImpl.java | 5 +++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java b/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java index e91477ed074..6a66ea3888a 100644 --- a/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java +++ b/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java @@ -79,6 +79,7 @@ import com.cloud.exception.ResourceAllocationException; import com.cloud.host.Host; import com.cloud.host.dao.HostDao; import com.cloud.storage.DataStoreRole; +import com.cloud.storage.RegisterVolumePayload; import com.cloud.storage.ScopeType; import com.cloud.storage.Storage.StoragePoolType; import com.cloud.storage.StoragePool; @@ -1404,7 +1405,7 @@ public class VolumeServiceImpl implements VolumeService { volumeStore.setDownloadState(Status.DOWNLOAD_ERROR); String msg = "Volume " + volume.getUuid() + " is corrupted on image store "; volumeStore.setErrorString(msg); - s_logger.info("msg"); + s_logger.info(msg); if (volumeStore.getDownloadUrl() == null) { msg = "Volume (" + volume.getUuid() + ") with install path " + volInfo.getInstallPath() + @@ -1453,7 +1454,6 @@ public class VolumeServiceImpl implements VolumeService { if (volumeStore.getDownloadState() != Status.DOWNLOADED) { s_logger.info("Volume Sync did not find " + volume.getName() + " ready on image store " + storeId + ", will request download to start/resume shortly"); - toBeDownloaded.add(volumeStore); } } @@ -1476,8 +1476,15 @@ public class VolumeServiceImpl implements VolumeService { } s_logger.debug("Volume " + volumeHost.getVolumeId() + " needs to be downloaded to " + store.getName()); - // TODO: pass a callback later - VolumeInfo vol = volFactory.getVolume(volumeHost.getVolumeId()); + // reset volume status back to Allocated + VolumeObject vol = (VolumeObject)volFactory.getVolume(volumeHost.getVolumeId()); + vol.processEvent(Event.OperationFailed); // reset back volume status + // remove leftover volume_store_ref entry since re-download will create it again + _volumeStoreDao.remove(volumeHost.getId()); + // get an updated volumeVO + vol = (VolumeObject)volFactory.getVolume(volumeHost.getVolumeId()); + RegisterVolumePayload payload = new RegisterVolumePayload(volumeHost.getDownloadUrl(), volumeHost.getChecksum(), vol.getFormat().toString()); + vol.addPayload(payload); createVolumeAsync(vol, store); } } diff --git a/server/src/com/cloud/storage/download/DownloadMonitorImpl.java b/server/src/com/cloud/storage/download/DownloadMonitorImpl.java index eed15811df7..f1937f85232 100755 --- a/server/src/com/cloud/storage/download/DownloadMonitorImpl.java +++ b/server/src/com/cloud/storage/download/DownloadMonitorImpl.java @@ -219,6 +219,11 @@ public class DownloadMonitorImpl extends ManagerBase implements DownloadMonitor _volumeStoreDao.persist(volumeHost); } else if ((volumeHost.getJobId() != null) && (volumeHost.getJobId().length() > 2)) { downloadJobExists = true; + } else { + // persit url and checksum + volumeHost.setDownloadUrl(url); + volumeHost.setChecksum(checkSum); + _volumeStoreDao.update(volumeHost.getId(), volumeHost); } Long maxVolumeSizeInBytes = getMaxVolumeSizeInBytes(); From 788fe5a27365e57508e0c1c967da46da0314e3e4 Mon Sep 17 00:00:00 2001 From: Koushik Das Date: Mon, 22 Dec 2014 10:52:13 +0530 Subject: [PATCH 012/391] CLOUDSTACK-8103: Vmsync marks VM as stopped even after failing to stop it in HV During vmsync if StopCommand (issued as part of PowerOff/PowerMissing report) fails to stop VM (since VM is running on HV), don't transition VM state to "Stopped" in CS db. Also added a check to throw ConcurrentOperationException if vm state is not "Running" after start operation. --- .../src/com/cloud/vm/VirtualMachineManagerImpl.java | 5 ++++- server/src/com/cloud/vm/UserVmManagerImpl.java | 7 +++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java index c87b7729b2e..ab5f6a3f046 100755 --- a/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java @@ -3688,7 +3688,10 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac VirtualMachineGuru vmGuru = getVmGuru(vm); VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm); - sendStop(vmGuru, profile, true, true); + if (!sendStop(vmGuru, profile, true, true)) { + // In case StopCommand fails, don't proceed further + return; + } try { stateTransitTo(vm, VirtualMachine.Event.FollowAgentPowerOffReport, null); diff --git a/server/src/com/cloud/vm/UserVmManagerImpl.java b/server/src/com/cloud/vm/UserVmManagerImpl.java index 4e64524fe71..3f539b25567 100755 --- a/server/src/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/com/cloud/vm/UserVmManagerImpl.java @@ -3194,6 +3194,13 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir try { vmParamPair = startVirtualMachine(vmId, hostId, additonalParams, deploymentPlannerToUse); vm = vmParamPair.first(); + + // At this point VM should be in "Running" state + UserVmVO tmpVm = _vmDao.findById(vm.getId()); + if (!tmpVm.getState().equals(State.Running)) { + // Some other thread changed state of VM, possibly vmsync + throw new ConcurrentOperationException("VM " + tmpVm + " unexpectedly went to " + tmpVm.getState() + " state"); + } } finally { updateVmStateForFailedVmCreation(vm.getId(), hostId); } From a2fa2e49113ad1a9cf921a621952b35bb8349655 Mon Sep 17 00:00:00 2001 From: Ashutosh K Date: Fri, 19 Dec 2014 12:51:07 +0530 Subject: [PATCH 013/391] CLOUDSTACK-8095: Fixed Iso attach issue in test_escalations_instances.py Signed-off-by: SrikanteswaraRao Talluri --- .../component/test_escalations_instances.py | 184 +++++++++--------- 1 file changed, 97 insertions(+), 87 deletions(-) diff --git a/test/integration/component/test_escalations_instances.py b/test/integration/component/test_escalations_instances.py index e79cd3d258a..c5afa274ba6 100644 --- a/test/integration/component/test_escalations_instances.py +++ b/test/integration/component/test_escalations_instances.py @@ -1577,8 +1577,7 @@ class TestListInstances(cloudstackTestCase): new_keypair = SSHKeyPair.register( self.userapiclient, name="keypair1", - publickey="ssh-rsa: e6:9a:1e:b5:98:75:88:5d:56:bc:92:7b:43:48:05:b2" - ) + publickey="ssh-rsa: e6:9a:1e:b5:98:75:88:5d:56:bc:92:7b:43:48:05:b2") self.assertIsNotNone( new_keypair, "New Key pair generation failed" @@ -2042,27 +2041,27 @@ class TestListInstances(cloudstackTestCase): ) status = validateList(list_vms_after) self.assertEquals( - PASS, - status[0], - "Listing of VM failed" + PASS, + status[0], + "Listing of VM failed" ) vm = list_vms_after[0] # Verifying that VM nics size is 1 now vm_nics_after = vm.nic self.assertIsNotNone( - vm_nics_after, - "Nic not found for the deployed VM" + vm_nics_after, + "Nic not found for the deployed VM" ) self.assertEquals( - 1, - len(vm_nics_after), - "VM NIC's count is not matching" + 1, + len(vm_nics_after), + "VM NIC's count is not matching" ) # Verifying the nic network is same as the default nic network self.assertEquals( - network2.id, - vm_nics_after[0].networkid, - "VM NIC is not same as expected" + network2.id, + vm_nics_after[0].networkid, + "VM NIC is not same as expected" ) return @@ -2125,23 +2124,23 @@ class TestInstances(cloudstackTestCase): self.apiClient = self.testClient.getApiClient() self.cleanup = [] self.account = Account.create( - self.apiClient, - self.services["account"], - domainid=self.domain.id - ) - # Getting authentication for user in newly created Account + self.apiClient, + self.services["account"], + domainid=self.domain.id + ) + # Getting authentication for user in newly created Account self.user = self.account.user[0] self.userapiclient = self.testClient.getUserApiClient( - self.user.username, - self.domain.name) + self.user.username, + self.domain.name) # Updating resource Limits for i in range(0, 8): Resources.updateLimit( - self.api_client, - account=self.account.name, - domainid=self.domain.id, - max=-1, - resourcetype=i + self.api_client, + account=self.account.name, + domainid=self.domain.id, + max=-1, + resourcetype=i ) self.cleanup.append(self.account) @@ -2239,6 +2238,7 @@ class TestInstances(cloudstackTestCase): vm_created, "VM creation failed" ) + self.cleanup.append(vm_created) # Listing all the VMs for a user again for matching zone list_vms_after = VirtualMachine.list( self.userapiclient, @@ -2268,60 +2268,70 @@ class TestInstances(cloudstackTestCase): isofilter="executable", zoneid=self.zone.id ) - # Verifying if size of the list is >= 1 - if isos_list is not None: - iso_toattach = isos_list[0] - # Attaching ISO listed to VM deployed - VirtualMachine.attach_iso( - vm_created, - self.userapiclient, - iso_toattach - ) - list_vm = VirtualMachine.list( - self.userapiclient, - id=vm_created.id - ) - status = validateList(list_vm) - self.assertEquals( - PASS, - status[0], - "VM listing by Id failed" - ) - # Verifying that attached ISO details are present in VM - self.assertEquals( - iso_toattach.name, - list_vm[0].isoname, - "Attached ISO name is not matching" - ) - self.assertEquals( - iso_toattach.displaytext, - list_vm[0].isodisplaytext, - "Attached ISO display is not matching" - ) - # Detaching ISO from VM - VirtualMachine.detach_iso( - vm_created, - self.userapiclient - ) - list_vm = VirtualMachine.list( - self.userapiclient, - id=vm_created.id - ) - status = validateList(list_vm) - self.assertEquals( - PASS, - status[0], - "VM listing by Id failed" - ) - # Verifying that ISO details are NOT present in VM - self.assertIsNone( - list_vm[0].isoname, - "ISO not detached from VM" - ) - else: - self.fail( - "Executable ISO in Ready is not found in the given setup") + self.assertEqual(validateList(isos_list)[0], + PASS, + "isos list validation failed") + + if self.hypervisor.lower() == "xenserver": + isoDesc = "xen" + else: + isoDesc = "vmware" + + validIsosToAttach = [iso for iso in isos_list + if isoDesc in iso.displaytext.lower()] + if not validIsosToAttach: + self.skipTest( + "Valid ISO not present in setup suitable to attach to VM") + + iso_toattach = validIsosToAttach[0] + # Attaching ISO listed to VM deployed + VirtualMachine.attach_iso( + vm_created, + self.userapiclient, + iso_toattach + ) + list_vm = VirtualMachine.list( + self.userapiclient, + id=vm_created.id + ) + status = validateList(list_vm) + self.assertEquals( + PASS, + status[0], + "VM listing by Id failed" + ) + # Verifying that attached ISO details are present in VM + self.assertEquals( + iso_toattach.name, + list_vm[0].isoname, + "Attached ISO name is not matching" + ) + self.assertEquals( + iso_toattach.displaytext, + list_vm[0].isodisplaytext, + "Attached ISO display is not matching" + ) + # Detaching ISO from VM + VirtualMachine.detach_iso( + vm_created, + self.userapiclient + ) + list_vm = VirtualMachine.list( + self.userapiclient, + id=vm_created.id + ) + status = validateList(list_vm) + self.assertEquals( + PASS, + status[0], + "VM listing by Id failed" + ) + # Verifying that ISO details are NOT present in VM + self.assertIsNone( + list_vm[0].isoname, + "ISO not detached from VM" + ) return @attr(tags=["advanced", "basic"], required_hardware="true") @@ -2866,9 +2876,9 @@ class TestInstances(cloudstackTestCase): ) self.assertEqual( - len(volumes), - len(list_volumes_page1) - len(list_data_disks), - "The volumes number should match with (volumes initially\ + len(volumes), + len(list_volumes_page1) - len(list_data_disks), + "The volumes number should match with (volumes initially\ present minus volumes detached") return @@ -2976,11 +2986,11 @@ class TestInstances(cloudstackTestCase): if service_offerings_list is not None: for i in range(0, len(service_offerings_list)): if not ((current_so.cpunumber > - service_offerings_list[i].cpunumber or + service_offerings_list[i].cpunumber or current_so.cpuspeed > - service_offerings_list[i].cpuspeed or + service_offerings_list[i].cpuspeed or current_so.memory > - service_offerings_list[i].memory) or + service_offerings_list[i].memory) or (current_so.cpunumber == service_offerings_list[i].cpunumber and current_so.cpuspeed == @@ -2989,9 +2999,9 @@ class TestInstances(cloudstackTestCase): service_offerings_list[i].memory)): if(current_so.storagetype == service_offerings_list[i].storagetype): - so_exists = True - new_so = service_offerings_list[i] - break + so_exists = True + new_so = service_offerings_list[i] + break # If service offering does not exists, then creating one service # offering for scale up if not so_exists: From 588e7dc8110e1cd03c87a97bd8ac39be0c0b1ed4 Mon Sep 17 00:00:00 2001 From: Gaurav Aradhye Date: Fri, 19 Dec 2014 18:59:53 +0530 Subject: [PATCH 014/391] CLOUDSTACK-8100: Fixed pep8 issues in test_vpc.py Signed-off-by: SrikanteswaraRao Talluri --- test/integration/component/test_vpc.py | 2577 ++++++++++++------------ 1 file changed, 1297 insertions(+), 1280 deletions(-) diff --git a/test/integration/component/test_vpc.py b/test/integration/component/test_vpc.py index 13f0eff35d8..58960a93712 100644 --- a/test/integration/component/test_vpc.py +++ b/test/integration/component/test_vpc.py @@ -17,159 +17,178 @@ """ Component tests for VPC functionality """ -#Import Local Modules +# Import Local Modules from nose.plugins.attrib import attr -from marvin.cloudstackTestCase import * +from marvin.cloudstackTestCase import cloudstackTestCase from marvin.cloudstackException import CloudstackAPIException -from marvin.cloudstackAPI import * -from marvin.lib.utils import * -from marvin.lib.base import * -from marvin.lib.common import * +from marvin.cloudstackAPI import updateZone +from marvin.lib.utils import cleanup_resources +from marvin.lib.base import (Account, + VPC, + VpcOffering, + VirtualMachine, + ServiceOffering, + Network, + NetworkOffering, + PublicIPAddress, + LoadBalancerRule, + Router, + NetworkACL, + NATRule, + Zone, + StaticNATRule) +from marvin.lib.common import (get_domain, + get_zone, + get_template, + list_configurations) +import time class Services: + """Test VPC services """ def __init__(self): self.services = { - "account": { - "email": "test@test.com", - "firstname": "Test", - "lastname": "User", - "username": "test", - # Random characters are appended for unique - # username - "password": "password", - }, - "domain_admin": { - "email": "domain@admin.com", - "firstname": "Domain", - "lastname": "Admin", - "username": "DoA", - # Random characters are appended for unique - # username - "password": "password", - }, - "service_offering": { - "name": "Tiny Instance", - "displaytext": "Tiny Instance", - "cpunumber": 1, - "cpuspeed": 100, - "memory": 128, - }, - "network_offering": { - "name": 'VPC Network offering', - "displaytext": 'VPC Network off', - "guestiptype": 'Isolated', - "supportedservices": 'Vpn,Dhcp,Dns,SourceNat,PortForwarding,Lb,UserData,StaticNat,NetworkACL', - "traffictype": 'GUEST', - "availability": 'Optional', - "useVpc": 'on', - "serviceProviderList": { - "Vpn": 'VpcVirtualRouter', - "Dhcp": 'VpcVirtualRouter', - "Dns": 'VpcVirtualRouter', - "SourceNat": 'VpcVirtualRouter', - "PortForwarding": 'VpcVirtualRouter', - "Lb": 'VpcVirtualRouter', - "UserData": 'VpcVirtualRouter', - "StaticNat": 'VpcVirtualRouter', - "NetworkACL": 'VpcVirtualRouter' - }, - }, - "network_offering_no_lb": { - "name": 'VPC Network offering', - "displaytext": 'VPC Network off', - "guestiptype": 'Isolated', - "supportedservices": 'Vpn,Dhcp,Dns,SourceNat,PortForwarding,UserData,StaticNat,NetworkACL', - "traffictype": 'GUEST', - "availability": 'Optional', - "useVpc": 'on', - "serviceProviderList": { - "Vpn": 'VpcVirtualRouter', - "Dhcp": 'VpcVirtualRouter', - "Dns": 'VpcVirtualRouter', - "SourceNat": 'VpcVirtualRouter', - "PortForwarding": 'VpcVirtualRouter', - "UserData": 'VpcVirtualRouter', - "StaticNat": 'VpcVirtualRouter', - "NetworkACL": 'VpcVirtualRouter' - }, - }, - "vpc_offering": { - "name": 'VPC off', - "displaytext": 'VPC off', - "supportedservices": 'Dhcp,Dns,SourceNat,PortForwarding,Vpn,Lb,UserData,StaticNat,NetworkACL', - }, - "vpc": { - "name": "TestVPC", - "displaytext": "TestVPC", - "cidr": '10.0.0.1/24' - }, - "vpc_no_name": { - "displaytext": "TestVPC", - "cidr": '10.0.0.1/24' - }, - "network": { - "name": "Test Network", - "displaytext": "Test Network", - "netmask": '255.255.255.0' - }, - "lbrule": { - "name": "SSH", - "alg": "leastconn", - # Algorithm used for load balancing - "privateport": 22, - "publicport": 2222, - "openfirewall": False, - "startport": 22, - "endport": 2222, - "protocol": "TCP", - "cidrlist": '0.0.0.0/0', - }, - "natrule": { - "privateport": 22, - "publicport": 22, - "startport": 22, - "endport": 22, - "protocol": "TCP", - "cidrlist": '0.0.0.0/0', - }, - "fw_rule": { - "startport": 1, - "endport": 6000, - "cidr": '0.0.0.0/0', - # Any network (For creating FW rule) - "protocol": "TCP" - }, - "icmp_rule": { - "icmptype": -1, - "icmpcode": -1, - "cidrlist": '0.0.0.0/0', - "protocol": "ICMP" - }, - "virtual_machine": { - "displayname": "Test VM", - "username": "root", - "password": "password", - "ssh_port": 22, - "hypervisor": 'XenServer', - # Hypervisor type should be same as - # hypervisor type of cluster - "privateport": 22, - "publicport": 22, - "protocol": 'TCP', - }, - "domain": { - "name": "TestDomain" - }, - "ostype": 'CentOS 5.3 (64-bit)', - # Cent OS 5.3 (64 bit) - "sleep": 60, - "timeout": 10, - "mode": 'advanced' - } + "account": { + "email": "test@test.com", + "firstname": "Test", + "lastname": "User", + "username": "test", + # Random characters are appended for unique + # username + "password": "password", + }, + "domain_admin": { + "email": "domain@admin.com", + "firstname": "Domain", + "lastname": "Admin", + "username": "DoA", + # Random characters are appended for unique + # username + "password": "password", + }, + "service_offering": { + "name": "Tiny Instance", + "displaytext": "Tiny Instance", + "cpunumber": 1, + "cpuspeed": 100, + "memory": 128, + }, + "network_offering": { + "name": 'VPC Network offering', + "displaytext": 'VPC Network off', + "guestiptype": 'Isolated', + "supportedservices": 'Vpn,Dhcp,Dns,SourceNat,PortForwarding,Lb,UserData,StaticNat,NetworkACL', + "traffictype": 'GUEST', + "availability": 'Optional', + "useVpc": 'on', + "serviceProviderList": { + "Vpn": 'VpcVirtualRouter', + "Dhcp": 'VpcVirtualRouter', + "Dns": 'VpcVirtualRouter', + "SourceNat": 'VpcVirtualRouter', + "PortForwarding": 'VpcVirtualRouter', + "Lb": 'VpcVirtualRouter', + "UserData": 'VpcVirtualRouter', + "StaticNat": 'VpcVirtualRouter', + "NetworkACL": 'VpcVirtualRouter' + }, + }, + "network_offering_no_lb": { + "name": 'VPC Network offering', + "displaytext": 'VPC Network off', + "guestiptype": 'Isolated', + "supportedservices": 'Vpn,Dhcp,Dns,SourceNat,PortForwarding,UserData,StaticNat,NetworkACL', + "traffictype": 'GUEST', + "availability": 'Optional', + "useVpc": 'on', + "serviceProviderList": { + "Vpn": 'VpcVirtualRouter', + "Dhcp": 'VpcVirtualRouter', + "Dns": 'VpcVirtualRouter', + "SourceNat": 'VpcVirtualRouter', + "PortForwarding": 'VpcVirtualRouter', + "UserData": 'VpcVirtualRouter', + "StaticNat": 'VpcVirtualRouter', + "NetworkACL": 'VpcVirtualRouter' + }, + }, + "vpc_offering": { + "name": 'VPC off', + "displaytext": 'VPC off', + "supportedservices": 'Dhcp,Dns,SourceNat,PortForwarding,Vpn,Lb,UserData,StaticNat,NetworkACL', + }, + "vpc": { + "name": "TestVPC", + "displaytext": "TestVPC", + "cidr": '10.0.0.1/24' + }, + "vpc_no_name": { + "displaytext": "TestVPC", + "cidr": '10.0.0.1/24' + }, + "network": { + "name": "Test Network", + "displaytext": "Test Network", + "netmask": '255.255.255.0' + }, + "lbrule": { + "name": "SSH", + "alg": "leastconn", + # Algorithm used for load balancing + "privateport": 22, + "publicport": 2222, + "openfirewall": False, + "startport": 22, + "endport": 2222, + "protocol": "TCP", + "cidrlist": '0.0.0.0/0', + }, + "natrule": { + "privateport": 22, + "publicport": 22, + "startport": 22, + "endport": 22, + "protocol": "TCP", + "cidrlist": '0.0.0.0/0', + }, + "fw_rule": { + "startport": 1, + "endport": 6000, + "cidr": '0.0.0.0/0', + # Any network (For creating FW rule) + "protocol": "TCP" + }, + "icmp_rule": { + "icmptype": -1, + "icmpcode": -1, + "cidrlist": '0.0.0.0/0', + "protocol": "ICMP" + }, + "virtual_machine": { + "displayname": "Test VM", + "username": "root", + "password": "password", + "ssh_port": 22, + "hypervisor": 'XenServer', + # Hypervisor type should be same as + # hypervisor type of cluster + "privateport": 22, + "publicport": 22, + "protocol": 'TCP', + }, + "domain": { + "name": "TestDomain" + }, + "ostype": 'CentOS 5.3 (64-bit)', + # Cent OS 5.3 (64 bit) + "sleep": 60, + "timeout": 10, + "mode": 'advanced' + } + class TestVPC(cloudstackTestCase): @@ -183,31 +202,31 @@ class TestVPC(cloudstackTestCase): cls.domain = get_domain(cls.api_client) cls.zone = get_zone(cls.api_client, cls.testClient.getZoneForTests()) cls.template = get_template( - cls.api_client, - cls.zone.id, - cls.services["ostype"] - ) + cls.api_client, + cls.zone.id, + cls.services["ostype"] + ) cls.services["virtual_machine"]["zoneid"] = cls.zone.id cls.services["virtual_machine"]["template"] = cls.template.id cls.service_offering = ServiceOffering.create( - cls.api_client, - cls.services["service_offering"] - ) + cls.api_client, + cls.services["service_offering"] + ) cls.vpc_off = VpcOffering.create( - cls.api_client, - cls.services["vpc_offering"] - ) + cls.api_client, + cls.services["vpc_offering"] + ) cls.vpc_off.update(cls.api_client, state='Enabled') cls._cleanup = [ - cls.service_offering, - ] + cls.service_offering, + ] return @classmethod def tearDownClass(cls): try: - #Cleanup resources used + # Cleanup resources used cleanup_resources(cls.api_client, cls._cleanup) except Exception as e: raise Exception("Warning: Exception during cleanup : %s" % e) @@ -217,11 +236,11 @@ class TestVPC(cloudstackTestCase): self.apiclient = self.testClient.getApiClient() self.dbclient = self.testClient.getDbConnection() self.account = Account.create( - self.apiclient, - self.services["account"], - admin=True, - domainid=self.domain.id - ) + self.apiclient, + self.services["account"], + admin=True, + domainid=self.domain.id + ) self.cleanup = [] self.cleanup.insert(0, self.account) return @@ -231,7 +250,6 @@ class TestVPC(cloudstackTestCase): cleanup_resources(self.apiclient, self.cleanup) except Exception as e: self.debug("Warning: Exception during cleanup : %s" % e) - #raise Exception("Warning: Exception during cleanup : %s" % e) return def validate_vpc_offering(self, vpc_offering): @@ -239,22 +257,22 @@ class TestVPC(cloudstackTestCase): self.debug("Check if the VPC offering is created successfully?") vpc_offs = VpcOffering.list( - self.apiclient, - id=vpc_offering.id - ) + self.apiclient, + id=vpc_offering.id + ) self.assertEqual( - isinstance(vpc_offs, list), - True, - "List VPC offerings should return a valid list" - ) + isinstance(vpc_offs, list), + True, + "List VPC offerings should return a valid list" + ) self.assertEqual( - vpc_offering.name, - vpc_offs[0].name, - "Name of the VPC offering should match with listVPCOff data" - ) + vpc_offering.name, + vpc_offs[0].name, + "Name of the VPC offering should match with listVPCOff data" + ) self.debug( - "VPC offering is created successfully - %s" % - vpc_offering.name) + "VPC offering is created successfully - %s" % + vpc_offering.name) return def validate_vpc_network(self, network, state=None): @@ -262,29 +280,30 @@ class TestVPC(cloudstackTestCase): self.debug("Check if the VPC network is created successfully?") vpc_networks = VPC.list( - self.apiclient, - id=network.id - ) + self.apiclient, + id=network.id + ) self.assertEqual( - isinstance(vpc_networks, list), - True, - "List VPC network should return a valid list" - ) + isinstance(vpc_networks, list), + True, + "List VPC network should return a valid list" + ) self.assertEqual( - network.name, - vpc_networks[0].name, - "Name of the VPC network should match with listVPC data" - ) + network.name, + vpc_networks[0].name, + "Name of the VPC network should match with listVPC data" + ) if state: self.assertEqual( - vpc_networks[0].state, - state, + vpc_networks[0].state, + state, "VPC state should be '%s'" % state - ) + ) self.debug("VPC network validated - %s" % network.name) return - #list_vpc_apis should be the first case otherwise the vpc counts would be wrong + # list_vpc_apis should be the first case otherwise the vpc counts would be + # wrong @attr(tags=["advanced", "intervlan"], required_hardware="false") def test_01_list_vpc_apis(self): """ Test list VPC APIs @@ -306,162 +325,161 @@ class TestVPC(cloudstackTestCase): self.services["vpc"]["cidr"] = "10.1.1.1/16" self.debug("creating a VPC network in the account: %s" % - self.account.name) + self.account.name) vpc_1 = VPC.create( - self.apiclient, - self.services["vpc"], - vpcofferingid=self.vpc_off.id, - zoneid=self.zone.id, - account=self.account.name, - domainid=self.account.domainid - ) + self.apiclient, + self.services["vpc"], + vpcofferingid=self.vpc_off.id, + zoneid=self.zone.id, + account=self.account.name, + domainid=self.account.domainid + ) self.validate_vpc_network(vpc_1) self.services["vpc"]["cidr"] = "10.1.46.1/16" vpc_2 = VPC.create( - self.apiclient, - self.services["vpc"], - vpcofferingid=self.vpc_off.id, - zoneid=self.zone.id, - account=self.account.name, - domainid=self.account.domainid - ) + self.apiclient, + self.services["vpc"], + vpcofferingid=self.vpc_off.id, + zoneid=self.zone.id, + account=self.account.name, + domainid=self.account.domainid + ) self.validate_vpc_network(vpc_2) self.debug("Check list VPC API by Name?") vpcs = VPC.list( - self.apiclient, - name=vpc_1.name, - listall=True - ) + self.apiclient, + name=vpc_1.name, + listall=True + ) self.assertEqual( - isinstance(vpcs, list), - True, - "List VPC shall return a valid resposne" - ) + isinstance(vpcs, list), + True, + "List VPC shall return a valid resposne" + ) vpc = vpcs[0] self.assertEqual( - vpc.name, - vpc_1.name, - "VPC name should match with the existing one" - ) + vpc.name, + vpc_1.name, + "VPC name should match with the existing one" + ) self.debug("Check list VPC API by displayText?") vpcs = VPC.list( - self.apiclient, - displaytext=vpc_1.displaytext, - listall=True - ) + self.apiclient, + displaytext=vpc_1.displaytext, + listall=True + ) self.assertEqual( - isinstance(vpcs, list), - True, - "List VPC shall return a valid resposne" - ) + isinstance(vpcs, list), + True, + "List VPC shall return a valid resposne" + ) vpc = vpcs[0] self.assertEqual( - vpc.displaytext, - vpc_1.displaytext, - "VPC displaytext should match with the existing one" - ) + vpc.displaytext, + vpc_1.displaytext, + "VPC displaytext should match with the existing one" + ) self.debug("Check list VPC API by cidr?") vpcs = VPC.list( - self.apiclient, - cidr=vpc_2.cidr, - listall=True - ) + self.apiclient, + cidr=vpc_2.cidr, + listall=True + ) self.assertEqual( - isinstance(vpcs, list), - True, - "List VPC shall return a valid resposne" - ) + isinstance(vpcs, list), + True, + "List VPC shall return a valid resposne" + ) vpc = vpcs[0] self.assertEqual( - vpc.cidr, - vpc_2.cidr, - "VPC cidr should match with the existing one" - ) + vpc.cidr, + vpc_2.cidr, + "VPC cidr should match with the existing one" + ) self.debug("Validating list VPC by Id") self.validate_vpc_network(vpc_1) self.debug("Validating list VPC by vpcofferingId") vpcs = VPC.list( - self.apiclient, - vpcofferingid=self.vpc_off.id, - listall=True - ) + self.apiclient, + vpcofferingid=self.vpc_off.id, + listall=True + ) self.assertEqual( - isinstance(vpcs, list), - True, - "List VPC by vpcofferingId should return a valid response" - ) + isinstance(vpcs, list), + True, + "List VPC by vpcofferingId should return a valid response" + ) self.debug("Length of list VPC response: %s" % len(vpcs)) self.assertEqual( - len(vpcs), - 2, - "List VPC should return 2 enabled VPCs" - ) + len(vpcs), + 2, + "List VPC should return 2 enabled VPCs" + ) for vpc in vpcs: self.assertEqual( - vpc.vpcofferingid, - self.vpc_off.id, - "VPC offering ID should match with that of resposne" - ) + vpc.vpcofferingid, + self.vpc_off.id, + "VPC offering ID should match with that of resposne" + ) self.debug("Validating list VPC by supportedservices") vpcs = VPC.list( - self.apiclient, - supportedservices='Vpn,Dhcp,Dns,SourceNat,PortForwarding,Lb,UserData,StaticNat,NetworkACL', - listall=True, - account=self.account.name, - domainid=self.account.domainid - ) + self.apiclient, + supportedservices='Vpn,Dhcp,Dns,SourceNat,PortForwarding,Lb,UserData,StaticNat,NetworkACL', + listall=True, + account=self.account.name, + domainid=self.account.domainid) self.assertEqual( - isinstance(vpcs, list), - True, - "List VPC by vpcofferingId should return a valid response" - ) + isinstance(vpcs, list), + True, + "List VPC by vpcofferingId should return a valid response" + ) for vpc in vpcs: self.assertIn( - vpc.id, - [vpc_1.id, vpc_2.id], - "VPC offering ID should match with that of resposne" - ) + vpc.id, + [vpc_1.id, vpc_2.id], + "VPC offering ID should match with that of resposne" + ) self.debug("Validating list VPC by restart required") vpcs = VPC.list( - self.apiclient, - restartrequired=True, - listall=True, - account=self.account.name, - domainid=self.account.domainid - ) + self.apiclient, + restartrequired=True, + listall=True, + account=self.account.name, + domainid=self.account.domainid + ) if vpcs is not None: for vpc in vpcs: self.assertEqual( - vpc.restartrequired, - True, - "RestartRequired should be set as True" - ) + vpc.restartrequired, + True, + "RestartRequired should be set as True" + ) self.debug("Validating list VPC by restart required") vpcs = VPC.list( - self.apiclient, - restartrequired=False, - listall=True, - account=self.account.name, - domainid=self.account.domainid - ) + self.apiclient, + restartrequired=False, + listall=True, + account=self.account.name, + domainid=self.account.domainid + ) self.assertEqual( - isinstance(vpcs, list), - True, - "List VPC by vpcofferingId should return a valid response" - ) + isinstance(vpcs, list), + True, + "List VPC by vpcofferingId should return a valid response" + ) if vpcs is not None: for vpc in vpcs: self.assertEqual( - vpc.restartrequired, - False, - "RestartRequired should be set as False" - ) + vpc.restartrequired, + False, + "RestartRequired should be set as False" + ) return @attr(tags=["advanced", "intervlan"], required_hardware="false") @@ -475,15 +493,15 @@ class TestVPC(cloudstackTestCase): self.services["vpc"]["cidr"] = "10.1.1.1/16" self.debug("creating a VPC network in the account: %s" % - self.account.name) + self.account.name) vpc = VPC.create( - self.apiclient, - self.services["vpc"], - vpcofferingid=self.vpc_off.id, - zoneid=self.zone.id, - account=self.account.name, - domainid=self.account.domainid - ) + self.apiclient, + self.services["vpc"], + vpcofferingid=self.vpc_off.id, + zoneid=self.zone.id, + account=self.account.name, + domainid=self.account.domainid + ) self.validate_vpc_network(vpc) self.debug("Restarting the VPC with no network") @@ -507,22 +525,22 @@ class TestVPC(cloudstackTestCase): self.services["vpc"]["cidr"] = "10.1.1.1/16" self.debug("creating a VPC network in the account: %s" % - self.account.name) + self.account.name) vpc = VPC.create( - self.apiclient, - self.services["vpc"], - vpcofferingid=self.vpc_off.id, - zoneid=self.zone.id, - account=self.account.name, - domainid=self.account.domainid - ) + self.apiclient, + self.services["vpc"], + vpcofferingid=self.vpc_off.id, + zoneid=self.zone.id, + account=self.account.name, + domainid=self.account.domainid + ) self.validate_vpc_network(vpc) self.network_offering = NetworkOffering.create( - self.apiclient, - self.services["network_offering"], - conservemode=False - ) + self.apiclient, + self.services["network_offering"], + conservemode=False + ) # Enable Network offering self.network_offering.update(self.apiclient, state='Enabled') self.cleanup.append(self.network_offering) @@ -534,41 +552,41 @@ class TestVPC(cloudstackTestCase): # Creating network using the network offering created self.debug("Creating network with network offering: %s" % - self.network_offering.id) + self.network_offering.id) network_1 = Network.create( - self.apiclient, - self.services["network"], - accountid=self.account.name, - domainid=self.account.domainid, - networkofferingid=self.network_offering.id, - zoneid=self.zone.id, - gateway=gateway, - vpcid=vpc.id - ) + self.apiclient, + self.services["network"], + accountid=self.account.name, + domainid=self.account.domainid, + networkofferingid=self.network_offering.id, + zoneid=self.zone.id, + gateway=gateway, + vpcid=vpc.id + ) self.debug("Created network with ID: %s" % network_1.id) self.network_offering_no_lb = NetworkOffering.create( - self.apiclient, - self.services["network_offering_no_lb"], - conservemode=False - ) + self.apiclient, + self.services["network_offering_no_lb"], + conservemode=False + ) # Enable Network offering self.network_offering_no_lb.update(self.apiclient, state='Enabled') self.cleanup.append(self.network_offering_no_lb) gateway = '10.1.2.1' # New network -> different gateway self.debug("Creating network with network offering: %s" % - self.network_offering_no_lb.id) + self.network_offering_no_lb.id) network_2 = Network.create( - self.apiclient, - self.services["network"], - accountid=self.account.name, - domainid=self.account.domainid, - networkofferingid=self.network_offering_no_lb.id, - zoneid=self.zone.id, - gateway=gateway, - vpcid=vpc.id - ) + self.apiclient, + self.services["network"], + accountid=self.account.name, + domainid=self.account.domainid, + networkofferingid=self.network_offering_no_lb.id, + zoneid=self.zone.id, + gateway=gateway, + vpcid=vpc.id + ) self.debug("Created network with ID: %s" % network_2.id) self.debug("Restarting the VPC with no network") @@ -591,15 +609,15 @@ class TestVPC(cloudstackTestCase): self.services["vpc"]["cidr"] = "10.1.1.1/16" self.debug("creating a VPC network in the account: %s" % - self.account.name) + self.account.name) vpc = VPC.create( - self.apiclient, - self.services["vpc"], - vpcofferingid=self.vpc_off.id, - zoneid=self.zone.id, - account=self.account.name, - domainid=self.account.domainid - ) + self.apiclient, + self.services["vpc"], + vpcofferingid=self.vpc_off.id, + zoneid=self.zone.id, + account=self.account.name, + domainid=self.account.domainid + ) self.validate_vpc_network(vpc) self.debug("Restarting the VPC with no network") @@ -610,14 +628,14 @@ class TestVPC(cloudstackTestCase): self.debug("Check if the VPC offering is deleted successfully?") vpcs = VPC.list( - self.apiclient, - id=vpc.id - ) + self.apiclient, + id=vpc.id + ) self.assertEqual( - vpcs, - None, - "List VPC offerings should not return anything" - ) + vpcs, + None, + "List VPC offerings should not return anything" + ) return @attr(tags=["advanced", "intervlan"], required_hardware="false") @@ -634,22 +652,22 @@ class TestVPC(cloudstackTestCase): self.services["vpc"]["cidr"] = "10.1.1.1/16" self.debug("creating a VPC network in the account: %s" % - self.account.name) + self.account.name) vpc = VPC.create( - self.apiclient, - self.services["vpc"], - vpcofferingid=self.vpc_off.id, - zoneid=self.zone.id, - account=self.account.name, - domainid=self.account.domainid - ) + self.apiclient, + self.services["vpc"], + vpcofferingid=self.vpc_off.id, + zoneid=self.zone.id, + account=self.account.name, + domainid=self.account.domainid + ) self.validate_vpc_network(vpc) self.network_offering = NetworkOffering.create( - self.apiclient, - self.services["network_offering"], - conservemode=False - ) + self.apiclient, + self.services["network_offering"], + conservemode=False + ) # Enable Network offering self.network_offering.update(self.apiclient, state='Enabled') self.cleanup.append(self.network_offering) @@ -661,41 +679,41 @@ class TestVPC(cloudstackTestCase): # Creating network using the network offering created self.debug("Creating network with network offering: %s" % - self.network_offering.id) + self.network_offering.id) network_1 = Network.create( - self.apiclient, - self.services["network"], - accountid=self.account.name, - domainid=self.account.domainid, - networkofferingid=self.network_offering.id, - zoneid=self.zone.id, - gateway=gateway, - vpcid=vpc.id - ) + self.apiclient, + self.services["network"], + accountid=self.account.name, + domainid=self.account.domainid, + networkofferingid=self.network_offering.id, + zoneid=self.zone.id, + gateway=gateway, + vpcid=vpc.id + ) self.debug("Created network with ID: %s" % network_1.id) self.network_offering_no_lb = NetworkOffering.create( - self.apiclient, - self.services["network_offering_no_lb"], - conservemode=False - ) + self.apiclient, + self.services["network_offering_no_lb"], + conservemode=False + ) # Enable Network offering self.network_offering_no_lb.update(self.apiclient, state='Enabled') self.cleanup.append(self.network_offering_no_lb) gateway = '10.1.2.1' # New network -> different gateway self.debug("Creating network with network offering: %s" % - self.network_offering_no_lb.id) + self.network_offering_no_lb.id) network_2 = Network.create( - self.apiclient, - self.services["network"], - accountid=self.account.name, - domainid=self.account.domainid, - networkofferingid=self.network_offering_no_lb.id, - zoneid=self.zone.id, - gateway=gateway, - vpcid=vpc.id - ) + self.apiclient, + self.services["network"], + accountid=self.account.name, + domainid=self.account.domainid, + networkofferingid=self.network_offering_no_lb.id, + zoneid=self.zone.id, + gateway=gateway, + vpcid=vpc.id + ) self.debug("Created network with ID: %s" % network_2.id) self.debug("Deleting the VPC with no network") @@ -718,37 +736,38 @@ class TestVPC(cloudstackTestCase): self.debug("Check if the VPC offering is deleted successfully?") vpcs = VPC.list( - self.apiclient, - id=vpc.id - ) + self.apiclient, + id=vpc.id + ) self.assertEqual( - vpcs, - None, - "List VPC offerings should not return anything" - ) - self.debug("Waiting for network.gc.interval to cleanup network resources") + vpcs, + None, + "List VPC offerings should not return anything" + ) + self.debug( + "Waiting for network.gc.interval to cleanup network resources") interval = list_configurations( - self.apiclient, - name='network.gc.interval' - ) + self.apiclient, + name='network.gc.interval' + ) wait = list_configurations( - self.apiclient, - name='network.gc.wait' - ) + self.apiclient, + name='network.gc.wait' + ) # Sleep to ensure that all resources are deleted time.sleep(int(interval[0].value) + int(wait[0].value)) self.debug("Check if VR is deleted or not?") routers = Router.list( - self.apiclient, - account=self.account.name, - domainid=self.account.domainid, - listall=True - ) + self.apiclient, + account=self.account.name, + domainid=self.account.domainid, + listall=True + ) self.assertEqual( - routers, - None, - "List Routers for the account should not return any response" - ) + routers, + None, + "List Routers for the account should not return any response" + ) return @attr(tags=["advanced", "intervlan"], required_hardware="false") @@ -762,53 +781,53 @@ class TestVPC(cloudstackTestCase): # 3. list VPCS as domain admin User to view all the Vpcs owned by admin self.user = Account.create( - self.apiclient, - self.services["account"], - ) + self.apiclient, + self.services["account"], + ) self.cleanup.append(self.user) self.services["vpc"]["cidr"] = "10.1.1.1/16" self.debug("creating a VPC network in the account: %s" % - self.account.name) + self.account.name) vpc_1 = VPC.create( - self.apiclient, - self.services["vpc"], - vpcofferingid=self.vpc_off.id, - zoneid=self.zone.id, - account=self.account.name, - domainid=self.account.domainid - ) + self.apiclient, + self.services["vpc"], + vpcofferingid=self.vpc_off.id, + zoneid=self.zone.id, + account=self.account.name, + domainid=self.account.domainid + ) self.validate_vpc_network(vpc_1) self.services["vpc"]["cidr"] = "10.1.46.1/16" vpc_2 = VPC.create( - self.apiclient, - self.services["vpc"], - vpcofferingid=self.vpc_off.id, - zoneid=self.zone.id, - account=self.user.name, - domainid=self.user.domainid - ) + self.apiclient, + self.services["vpc"], + vpcofferingid=self.vpc_off.id, + zoneid=self.zone.id, + account=self.user.name, + domainid=self.user.domainid + ) self.validate_vpc_network(vpc_2) self.debug("Validating list VPCs call by passing account and domain") vpcs = VPC.list( - self.apiclient, - account=self.user.name, - domainid=self.user.domainid, - listall=True - ) + self.apiclient, + account=self.user.name, + domainid=self.user.domainid, + listall=True + ) self.assertEqual( - isinstance(vpcs, list), - True, - "List VPC should return a valid response" - ) + isinstance(vpcs, list), + True, + "List VPC should return a valid response" + ) vpc = vpcs[0] self.assertEqual( - vpc.id, - vpc_2.id, - "List VPC should return VPC belonging to that account" - ) + vpc.id, + vpc_2.id, + "List VPC should return VPC belonging to that account" + ) return @attr(tags=["advanced", "intervlan", "multiple"], required_hardware="true") @@ -837,9 +856,9 @@ class TestVPC(cloudstackTestCase): self.debug("Creating a VPC offering..") vpc_off = VpcOffering.create( - self.apiclient, - self.services["vpc_offering"] - ) + self.apiclient, + self.services["vpc_offering"] + ) self.cleanup.append(vpc_off) self.validate_vpc_offering(vpc_off) @@ -848,249 +867,250 @@ class TestVPC(cloudstackTestCase): vpc_off.update(self.apiclient, state='Enabled') self.debug("creating a VPC network in the account: %s" % - self.account.name) + self.account.name) self.services["vpc"]["cidr"] = '10.1.1.1/16' vpc = VPC.create( - self.apiclient, - self.services["vpc"], - vpcofferingid=vpc_off.id, - zoneid=self.zone.id, - account=self.account.name, - domainid=self.account.domainid - ) + self.apiclient, + self.services["vpc"], + vpcofferingid=vpc_off.id, + zoneid=self.zone.id, + account=self.account.name, + domainid=self.account.domainid + ) self.validate_vpc_network(vpc) self.network_offering = NetworkOffering.create( - self.apiclient, - self.services["network_offering"], - conservemode=False - ) + self.apiclient, + self.services["network_offering"], + conservemode=False + ) # Enable Network offering self.network_offering.update(self.apiclient, state='Enabled') self.cleanup.append(self.network_offering) self.network_offering_no_lb = NetworkOffering.create( - self.apiclient, - self.services["network_offering_no_lb"], - conservemode=False - ) + self.apiclient, + self.services["network_offering_no_lb"], + conservemode=False + ) # Enable Network offering self.network_offering_no_lb.update(self.apiclient, state='Enabled') self.cleanup.append(self.network_offering_no_lb) # Creating network using the network offering created self.debug("Creating network with network offering: %s" % - self.network_offering_no_lb.id) + self.network_offering_no_lb.id) network_1 = Network.create( - self.apiclient, - self.services["network"], - accountid=self.account.name, - domainid=self.account.domainid, - networkofferingid=self.network_offering_no_lb.id, - zoneid=self.zone.id, - gateway='10.1.1.1', - vpcid=vpc.id - ) + self.apiclient, + self.services["network"], + accountid=self.account.name, + domainid=self.account.domainid, + networkofferingid=self.network_offering_no_lb.id, + zoneid=self.zone.id, + gateway='10.1.1.1', + vpcid=vpc.id + ) self.debug("Created network with ID: %s" % network_1.id) # Creating network using the network offering created self.debug("Creating network with network offering: %s" % - self.network_offering.id) + self.network_offering.id) network_2 = Network.create( - self.apiclient, - self.services["network"], - accountid=self.account.name, - domainid=self.account.domainid, - networkofferingid=self.network_offering.id, - zoneid=self.zone.id, - gateway='10.1.2.1', - vpcid=vpc.id - ) + self.apiclient, + self.services["network"], + accountid=self.account.name, + domainid=self.account.domainid, + networkofferingid=self.network_offering.id, + zoneid=self.zone.id, + gateway='10.1.2.1', + vpcid=vpc.id + ) self.debug("Created network with ID: %s" % network_2.id) self.debug("deploying VMs in network: %s" % network_1.name) # Spawn an instance in that network vm_1 = VirtualMachine.create( - self.apiclient, - self.services["virtual_machine"], - accountid=self.account.name, - domainid=self.account.domainid, - serviceofferingid=self.service_offering.id, - networkids=[str(network_1.id)] - ) + self.apiclient, + self.services["virtual_machine"], + accountid=self.account.name, + domainid=self.account.domainid, + serviceofferingid=self.service_offering.id, + networkids=[str(network_1.id)] + ) self.debug("Deployed VM in network: %s" % network_1.id) vm_2 = VirtualMachine.create( - self.apiclient, - self.services["virtual_machine"], - accountid=self.account.name, - domainid=self.account.domainid, - serviceofferingid=self.service_offering.id, - networkids=[str(network_1.id)] - ) + self.apiclient, + self.services["virtual_machine"], + accountid=self.account.name, + domainid=self.account.domainid, + serviceofferingid=self.service_offering.id, + networkids=[str(network_1.id)] + ) self.debug("Deployed VM in network: %s" % network_1.id) self.debug("deploying VMs in network: %s" % network_2.name) # Spawn an instance in that network vm_3 = VirtualMachine.create( - self.apiclient, - self.services["virtual_machine"], - accountid=self.account.name, - domainid=self.account.domainid, - serviceofferingid=self.service_offering.id, - networkids=[str(network_2.id)] - ) + self.apiclient, + self.services["virtual_machine"], + accountid=self.account.name, + domainid=self.account.domainid, + serviceofferingid=self.service_offering.id, + networkids=[str(network_2.id)] + ) self.debug("Deployed VM in network: %s" % network_2.id) vm_4 = VirtualMachine.create( - self.apiclient, - self.services["virtual_machine"], - accountid=self.account.name, - domainid=self.account.domainid, - serviceofferingid=self.service_offering.id, - networkids=[str(network_2.id)] - ) + self.apiclient, + self.services["virtual_machine"], + accountid=self.account.name, + domainid=self.account.domainid, + serviceofferingid=self.service_offering.id, + networkids=[str(network_2.id)] + ) self.debug("Deployed VM in network: %s" % network_2.id) self.debug("Associating public IP for network: %s" % network_1.name) public_ip_1 = PublicIPAddress.create( - self.apiclient, - accountid=self.account.name, - zoneid=self.zone.id, - domainid=self.account.domainid, - networkid=network_1.id, - vpcid=vpc.id - ) + self.apiclient, + accountid=self.account.name, + zoneid=self.zone.id, + domainid=self.account.domainid, + networkid=network_1.id, + vpcid=vpc.id + ) self.debug("Associated %s with network %s" % ( - public_ip_1.ipaddress.ipaddress, - network_1.id - )) + public_ip_1.ipaddress.ipaddress, + network_1.id + )) - nat_rule = NATRule.create( - self.apiclient, - vm_1, - self.services["natrule"], - ipaddressid=public_ip_1.ipaddress.id, - openfirewall=False, - networkid=network_1.id, - vpcid=vpc.id - ) + NATRule.create( + self.apiclient, + vm_1, + self.services["natrule"], + ipaddressid=public_ip_1.ipaddress.id, + openfirewall=False, + networkid=network_1.id, + vpcid=vpc.id + ) self.debug("Adding NetwrokACl rules to make NAT rule accessible") - nwacl_nat = NetworkACL.create( - self.apiclient, - networkid=network_1.id, - services=self.services["natrule"], - traffictype='Ingress' - ) + NetworkACL.create( + self.apiclient, + networkid=network_1.id, + services=self.services["natrule"], + traffictype='Ingress' + ) self.debug("Associating public IP for network: %s" % network_1.name) public_ip_2 = PublicIPAddress.create( - self.apiclient, - accountid=self.account.name, - zoneid=self.zone.id, - domainid=self.account.domainid, - networkid=network_1.id, - vpcid=vpc.id - ) + self.apiclient, + accountid=self.account.name, + zoneid=self.zone.id, + domainid=self.account.domainid, + networkid=network_1.id, + vpcid=vpc.id + ) self.debug("Associated %s with network %s" % ( - public_ip_2.ipaddress.ipaddress, - network_1.id - )) + public_ip_2.ipaddress.ipaddress, + network_1.id + )) self.debug("Enabling static NAT for IP: %s" % - public_ip_2.ipaddress.ipaddress) + public_ip_2.ipaddress.ipaddress) try: StaticNATRule.enable( - self.apiclient, - ipaddressid=public_ip_2.ipaddress.id, - virtualmachineid=vm_2.id, - networkid=network_1.id - ) + self.apiclient, + ipaddressid=public_ip_2.ipaddress.id, + virtualmachineid=vm_2.id, + networkid=network_1.id + ) self.debug("Static NAT enabled for IP: %s" % - public_ip_2.ipaddress.ipaddress) + public_ip_2.ipaddress.ipaddress) except Exception as e: self.fail("Failed to enable static NAT on IP: %s - %s" % ( - public_ip_2.ipaddress.ipaddress, e)) + public_ip_2.ipaddress.ipaddress, e)) public_ips = PublicIPAddress.list( - self.apiclient, - networkid=network_1.id, - listall=True, - isstaticnat=True, - account=self.account.name, - domainid=self.account.domainid - ) + self.apiclient, + networkid=network_1.id, + listall=True, + isstaticnat=True, + account=self.account.name, + domainid=self.account.domainid + ) self.assertEqual( - isinstance(public_ips, list), - True, - "List public Ip for network should list the Ip addr" - ) + isinstance(public_ips, list), + True, + "List public Ip for network should list the Ip addr" + ) self.assertEqual( - public_ips[0].ipaddress, - public_ip_2.ipaddress.ipaddress, - "List public Ip for network should list the Ip addr" - ) + public_ips[0].ipaddress, + public_ip_2.ipaddress.ipaddress, + "List public Ip for network should list the Ip addr" + ) self.debug("Associating public IP for network: %s" % vpc.name) public_ip_3 = PublicIPAddress.create( - self.apiclient, - accountid=self.account.name, - zoneid=self.zone.id, - domainid=self.account.domainid, - networkid=network_2.id, - vpcid=vpc.id - ) + self.apiclient, + accountid=self.account.name, + zoneid=self.zone.id, + domainid=self.account.domainid, + networkid=network_2.id, + vpcid=vpc.id + ) self.debug("Associated %s with network %s" % ( - public_ip_3.ipaddress.ipaddress, - network_2.id - )) + public_ip_3.ipaddress.ipaddress, + network_2.id + )) self.debug("Creating LB rule for IP address: %s" % - public_ip_3.ipaddress.ipaddress) + public_ip_3.ipaddress.ipaddress) lb_rule = LoadBalancerRule.create( - self.apiclient, - self.services["lbrule"], - ipaddressid=public_ip_3.ipaddress.id, - accountid=self.account.name, - networkid=network_2.id, - vpcid=vpc.id, - domainid=self.account.domainid - ) + self.apiclient, + self.services["lbrule"], + ipaddressid=public_ip_3.ipaddress.id, + accountid=self.account.name, + networkid=network_2.id, + vpcid=vpc.id, + domainid=self.account.domainid + ) self.debug("Adding virtual machines %s and %s to LB rule" % ( - vm_3.name, vm_4.name)) + vm_3.name, vm_4.name)) lb_rule.assign(self.apiclient, [vm_3, vm_4]) self.debug("Adding NetwrokACl rules to make PF and LB accessible") - nwacl_lb = NetworkACL.create( - self.apiclient, - networkid=network_2.id, - services=self.services["lbrule"], - traffictype='Ingress' - ) + NetworkACL.create( + self.apiclient, + networkid=network_2.id, + services=self.services["lbrule"], + traffictype='Ingress' + ) - self.debug("Adding Egress rules to network %s and %s to allow access to internet") - nwacl_internet_1 = NetworkACL.create( - self.apiclient, - networkid=network_1.id, - services=self.services["icmp_rule"], - traffictype='Egress' - ) - nwacl_internet_2 = NetworkACL.create( - self.apiclient, - networkid=network_2.id, - services=self.services["icmp_rule"], - traffictype='Egress' - ) + self.debug("Adding Egress rules to network %s and %s to allow\ + access to internet") + NetworkACL.create( + self.apiclient, + networkid=network_1.id, + services=self.services["icmp_rule"], + traffictype='Egress' + ) + NetworkACL.create( + self.apiclient, + networkid=network_2.id, + services=self.services["icmp_rule"], + traffictype='Egress' + ) self.debug("Checking if we can SSH into VM_1?") try: ssh_1 = vm_1.get_ssh_client( - ipaddress=public_ip_1.ipaddress.ipaddress, - reconnect=True, - port=self.services["natrule"]["publicport"] - ) + ipaddress=public_ip_1.ipaddress.ipaddress, + reconnect=True, + port=self.services["natrule"]["publicport"] + ) self.debug("SSH into VM is successfully") self.debug("Verifying if we can ping to outside world from VM?") @@ -1103,61 +1123,61 @@ class TestVPC(cloudstackTestCase): # rtt min/avg/max/mdev = 25.970/25.970/25.970/0.000 ms except Exception as e: self.fail("Failed to SSH into VM - %s, %s" % - (public_ip_1.ipaddress.ipaddress, e)) + (public_ip_1.ipaddress.ipaddress, e)) result = str(res) self.debug("Result: %s" % result) self.assertEqual( - result.count("1 received"), - 1, - "Ping to outside world from VM should be successful" - ) + result.count("1 received"), + 1, + "Ping to outside world from VM should be successful" + ) self.debug("Checking if we can SSH into VM_2?") try: ssh_2 = vm_2.get_ssh_client( - ipaddress=public_ip_2.ipaddress.ipaddress, - reconnect=True, - port=self.services["natrule"]["publicport"] - ) + ipaddress=public_ip_2.ipaddress.ipaddress, + reconnect=True, + port=self.services["natrule"]["publicport"] + ) self.debug("SSH into VM is successfully") self.debug("Verifying if we can ping to outside world from VM?") res = ssh_2.execute("ping -c 1 www.google.com") except Exception as e: self.fail("Failed to SSH into VM - %s, %s" % - (public_ip_2.ipaddress.ipaddress, e)) + (public_ip_2.ipaddress.ipaddress, e)) result = str(res) self.debug("Result: %s" % result) self.assertEqual( - result.count("1 received"), - 1, - "Ping to outside world from VM should be successful" - ) + result.count("1 received"), + 1, + "Ping to outside world from VM should be successful" + ) self.debug("Checking if we can SSH into VM using LB rule?") try: ssh_3 = vm_3.get_ssh_client( - ipaddress=public_ip_3.ipaddress.ipaddress, - reconnect=True, - port=self.services["lbrule"]["publicport"] - ) + ipaddress=public_ip_3.ipaddress.ipaddress, + reconnect=True, + port=self.services["lbrule"]["publicport"] + ) self.debug("SSH into VM is successfully") self.debug("Verifying if we can ping to outside world from VM?") res = ssh_3.execute("ping -c 1 www.google.com") except Exception as e: self.fail("Failed to SSH into VM - %s, %s" % - (public_ip_3.ipaddress.ipaddress, e)) + (public_ip_3.ipaddress.ipaddress, e)) result = str(res) self.debug("Result: %s" % result) self.assertEqual( - result.count("1 received"), - 1, - "Ping to outside world from VM should be successful" - ) + result.count("1 received"), + 1, + "Ping to outside world from VM should be successful" + ) return @attr(tags=["advanced", "intervlan"], required_hardware="true") @@ -1182,9 +1202,9 @@ class TestVPC(cloudstackTestCase): self.cleanup = [] self.debug("Creating a VPC offering..") vpc_off = VpcOffering.create( - self.apiclient, - self.services["vpc_offering"] - ) + self.apiclient, + self.services["vpc_offering"] + ) self.cleanup.append(vpc_off) self.validate_vpc_offering(vpc_off) @@ -1193,248 +1213,250 @@ class TestVPC(cloudstackTestCase): vpc_off.update(self.apiclient, state='Enabled') self.debug("creating a VPC network in the account: %s" % - self.account.name) + self.account.name) self.services["vpc"]["cidr"] = '10.1.1.1/16' vpc = VPC.create( - self.apiclient, - self.services["vpc"], - vpcofferingid=vpc_off.id, - zoneid=self.zone.id, - account=self.account.name, - domainid=self.account.domainid - ) + self.apiclient, + self.services["vpc"], + vpcofferingid=vpc_off.id, + zoneid=self.zone.id, + account=self.account.name, + domainid=self.account.domainid + ) self.validate_vpc_network(vpc) self.network_offering = NetworkOffering.create( - self.apiclient, - self.services["network_offering"], - conservemode=False - ) + self.apiclient, + self.services["network_offering"], + conservemode=False + ) # Enable Network offering self.network_offering.update(self.apiclient, state='Enabled') self.cleanup.append(self.network_offering) self.network_offering_no_lb = NetworkOffering.create( - self.apiclient, - self.services["network_offering_no_lb"], - conservemode=False - ) + self.apiclient, + self.services["network_offering_no_lb"], + conservemode=False + ) # Enable Network offering self.network_offering_no_lb.update(self.apiclient, state='Enabled') self.cleanup.append(self.network_offering_no_lb) # Creating network using the network offering created self.debug("Creating network with network offering: %s" % - self.network_offering.id) + self.network_offering.id) network_1 = Network.create( - self.apiclient, - self.services["network"], - accountid=self.account.name, - domainid=self.account.domainid, - networkofferingid=self.network_offering_no_lb.id, - zoneid=self.zone.id, - gateway='10.1.1.1', - vpcid=vpc.id - ) + self.apiclient, + self.services["network"], + accountid=self.account.name, + domainid=self.account.domainid, + networkofferingid=self.network_offering_no_lb.id, + zoneid=self.zone.id, + gateway='10.1.1.1', + vpcid=vpc.id + ) self.debug("Created network with ID: %s" % network_1.id) # Creating network using the network offering created self.debug("Creating network with network offering: %s" % - self.network_offering_no_lb.id) + self.network_offering_no_lb.id) network_2 = Network.create( - self.apiclient, - self.services["network"], - accountid=self.account.name, - domainid=self.account.domainid, - networkofferingid=self.network_offering.id, - zoneid=self.zone.id, - gateway='10.1.2.1', - vpcid=vpc.id - ) + self.apiclient, + self.services["network"], + accountid=self.account.name, + domainid=self.account.domainid, + networkofferingid=self.network_offering.id, + zoneid=self.zone.id, + gateway='10.1.2.1', + vpcid=vpc.id + ) self.debug("Created network with ID: %s" % network_2.id) self.debug("deploying VMs in network: %s" % network_1.name) # Spawn an instance in that network vm_1 = VirtualMachine.create( - self.apiclient, - self.services["virtual_machine"], - accountid=self.account.name, - domainid=self.account.domainid, - serviceofferingid=self.service_offering.id, - networkids=[str(network_1.id)] - ) + self.apiclient, + self.services["virtual_machine"], + accountid=self.account.name, + domainid=self.account.domainid, + serviceofferingid=self.service_offering.id, + networkids=[str(network_1.id)] + ) self.debug("Deployed VM in network: %s" % network_1.id) vm_2 = VirtualMachine.create( - self.apiclient, - self.services["virtual_machine"], - accountid=self.account.name, - domainid=self.account.domainid, - serviceofferingid=self.service_offering.id, - networkids=[str(network_1.id)] - ) + self.apiclient, + self.services["virtual_machine"], + accountid=self.account.name, + domainid=self.account.domainid, + serviceofferingid=self.service_offering.id, + networkids=[str(network_1.id)] + ) self.debug("Deployed VM in network: %s" % network_1.id) self.debug("deploying VMs in network: %s" % network_2.name) # Spawn an instance in that network vm_3 = VirtualMachine.create( - self.apiclient, - self.services["virtual_machine"], - accountid=self.account.name, - domainid=self.account.domainid, - serviceofferingid=self.service_offering.id, - networkids=[str(network_2.id)] - ) + self.apiclient, + self.services["virtual_machine"], + accountid=self.account.name, + domainid=self.account.domainid, + serviceofferingid=self.service_offering.id, + networkids=[str(network_2.id)] + ) self.debug("Deployed VM in network: %s" % network_2.id) vm_4 = VirtualMachine.create( - self.apiclient, - self.services["virtual_machine"], - accountid=self.account.name, - domainid=self.account.domainid, - serviceofferingid=self.service_offering.id, - networkids=[str(network_2.id)] - ) + self.apiclient, + self.services["virtual_machine"], + accountid=self.account.name, + domainid=self.account.domainid, + serviceofferingid=self.service_offering.id, + networkids=[str(network_2.id)] + ) self.debug("Deployed VM in network: %s" % network_2.id) self.debug("Associating public IP for network: %s" % network_1.name) public_ip_1 = PublicIPAddress.create( - self.apiclient, - accountid=self.account.name, - zoneid=self.zone.id, - domainid=self.account.domainid, - networkid=network_1.id, - vpcid=vpc.id - ) + self.apiclient, + accountid=self.account.name, + zoneid=self.zone.id, + domainid=self.account.domainid, + networkid=network_1.id, + vpcid=vpc.id + ) self.debug("Associated %s with network %s" % ( - public_ip_1.ipaddress.ipaddress, - network_1.id - )) + public_ip_1.ipaddress.ipaddress, + network_1.id + )) - nat_rule = NATRule.create( - self.apiclient, - vm_1, - self.services["natrule"], - ipaddressid=public_ip_1.ipaddress.id, - openfirewall=False, - networkid=network_1.id, - vpcid=vpc.id - ) + NATRule.create( + self.apiclient, + vm_1, + self.services["natrule"], + ipaddressid=public_ip_1.ipaddress.id, + openfirewall=False, + networkid=network_1.id, + vpcid=vpc.id + ) self.debug("Adding NetwrokACl rules to make NAT rule accessible") - nwacl_nat = NetworkACL.create( - self.apiclient, - networkid=network_1.id, - services=self.services["natrule"], - traffictype='Ingress' - ) + NetworkACL.create( + self.apiclient, + networkid=network_1.id, + services=self.services["natrule"], + traffictype='Ingress' + ) self.debug("Associating public IP for network: %s" % network_1.name) public_ip_2 = PublicIPAddress.create( - self.apiclient, - accountid=self.account.name, - zoneid=self.zone.id, - domainid=self.account.domainid, - networkid=network_1.id, - vpcid=vpc.id - ) + self.apiclient, + accountid=self.account.name, + zoneid=self.zone.id, + domainid=self.account.domainid, + networkid=network_1.id, + vpcid=vpc.id + ) self.debug("Associated %s with network %s" % ( - public_ip_2.ipaddress.ipaddress, - network_1.id - )) + public_ip_2.ipaddress.ipaddress, + network_1.id + )) self.debug("Enabling static NAT for IP: %s" % - public_ip_2.ipaddress.ipaddress) + public_ip_2.ipaddress.ipaddress) try: StaticNATRule.enable( - self.apiclient, - ipaddressid=public_ip_2.ipaddress.id, - virtualmachineid=vm_2.id, - networkid=network_1.id - ) + self.apiclient, + ipaddressid=public_ip_2.ipaddress.id, + virtualmachineid=vm_2.id, + networkid=network_1.id + ) self.debug("Static NAT enabled for IP: %s" % - public_ip_2.ipaddress.ipaddress) + public_ip_2.ipaddress.ipaddress) except Exception as e: self.fail("Failed to enable static NAT on IP: %s - %s" % ( - public_ip_2.ipaddress.ipaddress, e)) + public_ip_2.ipaddress.ipaddress, e)) public_ips = PublicIPAddress.list( - self.apiclient, - networkid=network_1.id, - listall=True, - isstaticnat=True, - account=self.account.name, - domainid=self.account.domainid - ) + self.apiclient, + networkid=network_1.id, + listall=True, + isstaticnat=True, + account=self.account.name, + domainid=self.account.domainid + ) self.assertEqual( - isinstance(public_ips, list), - True, - "List public Ip for network should list the Ip addr" - ) + isinstance(public_ips, list), + True, + "List public Ip for network should list the Ip addr" + ) self.assertEqual( - public_ips[0].ipaddress, - public_ip_2.ipaddress.ipaddress, - "List public Ip for network should list the Ip addr" - ) + public_ips[0].ipaddress, + public_ip_2.ipaddress.ipaddress, + "List public Ip for network should list the Ip addr" + ) self.debug("Associating public IP for network: %s" % vpc.name) public_ip_3 = PublicIPAddress.create( - self.apiclient, - accountid=self.account.name, - zoneid=self.zone.id, - domainid=self.account.domainid, - networkid=network_2.id, - vpcid=vpc.id - ) + self.apiclient, + accountid=self.account.name, + zoneid=self.zone.id, + domainid=self.account.domainid, + networkid=network_2.id, + vpcid=vpc.id + ) self.debug("Associated %s with network %s" % ( - public_ip_3.ipaddress.ipaddress, - network_2.id - )) + public_ip_3.ipaddress.ipaddress, + network_2.id + )) self.debug("Creating LB rule for IP address: %s" % - public_ip_3.ipaddress.ipaddress) + public_ip_3.ipaddress.ipaddress) lb_rule = LoadBalancerRule.create( - self.apiclient, - self.services["lbrule"], - ipaddressid=public_ip_3.ipaddress.id, - accountid=self.account.name, - networkid=network_2.id, - vpcid=vpc.id, - domainid=self.account.domainid - ) + self.apiclient, + self.services["lbrule"], + ipaddressid=public_ip_3.ipaddress.id, + accountid=self.account.name, + networkid=network_2.id, + vpcid=vpc.id, + domainid=self.account.domainid + ) self.debug("Adding virtual machines %s and %s to LB rule" % ( - vm_3.name, vm_4.name)) + vm_3.name, vm_4.name)) lb_rule.assign(self.apiclient, [vm_3, vm_4]) self.debug("Adding NetwrokACl rules to make PF and LB accessible") - nwacl_lb = NetworkACL.create( - self.apiclient, - networkid=network_2.id, - services=self.services["lbrule"], - traffictype='Ingress' - ) + NetworkACL.create( + self.apiclient, + networkid=network_2.id, + services=self.services["lbrule"], + traffictype='Ingress' + ) - self.debug("Adding Egress rules to network %s and %s to allow access to internet") - nwacl_internet_1 = NetworkACL.create( - self.apiclient, - networkid=network_1.id, - services=self.services["icmp_rule"], - traffictype='Egress' - ) - nwacl_internet_2 = NetworkACL.create( - self.apiclient, - networkid=network_2.id, - services=self.services["icmp_rule"], - traffictype='Egress' - ) + self.debug( + "Adding Egress rules to network %s and %s to allow\ + access to internet") + NetworkACL.create( + self.apiclient, + networkid=network_1.id, + services=self.services["icmp_rule"], + traffictype='Egress' + ) + NetworkACL.create( + self.apiclient, + networkid=network_2.id, + services=self.services["icmp_rule"], + traffictype='Egress' + ) self.debug("Checking if we can SSH into VM_1?") try: ssh_1 = vm_1.get_ssh_client( - ipaddress=public_ip_1.ipaddress.ipaddress, - reconnect=True, - port=self.services["natrule"]["publicport"]) + ipaddress=public_ip_1.ipaddress.ipaddress, + reconnect=True, + port=self.services["natrule"]["publicport"]) self.debug("SSH into VM is successfully") self.debug("Verifying if we can ping to outside world from VM?") @@ -1447,83 +1469,83 @@ class TestVPC(cloudstackTestCase): # rtt min/avg/max/mdev = 25.970/25.970/25.970/0.000 ms except Exception as e: self.fail("Failed to SSH into VM - %s, %s" % - (public_ip_1.ipaddress.ipaddress, e)) + (public_ip_1.ipaddress.ipaddress, e)) result = str(res) self.debug("result: %s" % result) self.assertEqual( - result.count("1 received"), - 1, - "Ping to outside world from VM should be successful" - ) + result.count("1 received"), + 1, + "Ping to outside world from VM should be successful" + ) self.debug("Checking if we can SSH into VM_2?") try: ssh_2 = vm_2.get_ssh_client( - ipaddress=public_ip_2.ipaddress.ipaddress, - reconnect=True, - port=self.services["natrule"]["publicport"]) + ipaddress=public_ip_2.ipaddress.ipaddress, + reconnect=True, + port=self.services["natrule"]["publicport"]) self.debug("SSH into VM is successfully") self.debug("Verifying if we can ping to outside world from VM?") res = ssh_2.execute("ping -c 1 www.google.com") except Exception as e: self.fail("Failed to SSH into VM - %s, %s" % - (public_ip_2.ipaddress.ipaddress, e)) + (public_ip_2.ipaddress.ipaddress, e)) result = str(res) self.debug("Result: %s" % result) self.assertEqual( - result.count("1 received"), - 1, - "Ping to outside world from VM should be successful" - ) + result.count("1 received"), + 1, + "Ping to outside world from VM should be successful" + ) self.debug("Checking if we can SSH into VM using LB rule?") try: ssh_3 = vm_3.get_ssh_client( - ipaddress=public_ip_3.ipaddress.ipaddress, - reconnect=True, - port=self.services["lbrule"]["publicport"] - ) + ipaddress=public_ip_3.ipaddress.ipaddress, + reconnect=True, + port=self.services["lbrule"]["publicport"] + ) self.debug("SSH into VM is successfully") self.debug("Verifying if we can ping to outside world from VM?") res = ssh_3.execute("ping -c 1 www.google.com") except Exception as e: self.fail("Failed to SSH into VM - %s, %s" % - (public_ip_3.ipaddress.ipaddress, e)) + (public_ip_3.ipaddress.ipaddress, e)) result = str(res) self.debug("Result: %s" % result) self.assertEqual( - result.count("1 received"), - 1, - "Ping to outside world from VM should be successful" - ) + result.count("1 received"), + 1, + "Ping to outside world from VM should be successful" + ) self.debug("Deleting the account") self.account.delete(self.apiclient) self.debug("Waiting for account to cleanup") interval = list_configurations( - self.apiclient, - name='account.cleanup.interval' - ) + self.apiclient, + name='account.cleanup.interval' + ) # Sleep to ensure that all resources are deleted time.sleep(int(interval[0].value)) self.debug("Checking if VPC is deleted after account deletion") vpcs = VPC.list( - self.apiclient, - id=vpc.id, - listall=True - ) + self.apiclient, + id=vpc.id, + listall=True + ) self.assertEqual( - vpcs, - None, - "List VPC should not return any response" - ) + vpcs, + None, + "List VPC should not return any response" + ) return @attr(tags=["advanced", "intervlan"], required_hardware="false") @@ -1538,54 +1560,54 @@ class TestVPC(cloudstackTestCase): self.services["vpc"]["cidr"] = "10.1.1.1/16" self.debug("creating a VPC network in the account: %s" % - self.account.name) + self.account.name) vpc = VPC.create( - self.apiclient, - self.services["vpc"], - vpcofferingid=self.vpc_off.id, - zoneid=self.zone.id, - account=self.account.name, - domainid=self.account.domainid - ) + self.apiclient, + self.services["vpc"], + vpcofferingid=self.vpc_off.id, + zoneid=self.zone.id, + account=self.account.name, + domainid=self.account.domainid + ) self.validate_vpc_network(vpc) self.debug("Verify if the VPC was created with enabled state") self.assertEqual( - vpc.state, - 'Enabled', - "VPC after creation should be in enabled state but the " - "state is %s" % vpc.state - ) + vpc.state, + 'Enabled', + "VPC after creation should be in enabled state but the " + "state is %s" % vpc.state + ) self.debug("Verify if the Router has started") routers = Router.list( - self.apiclient, - account=self.account.name, - domainid=self.account.domainid, - listall=True - ) + self.apiclient, + account=self.account.name, + domainid=self.account.domainid, + listall=True + ) self.assertEqual( - isinstance(routers, list), - True, - "List Routers should return a valid list" - ) + isinstance(routers, list), + True, + "List Routers should return a valid list" + ) self.assertEqual(routers[0].state, - 'Running', - "Router should be in running state" - ) + 'Running', + "Router should be in running state" + ) src_nat_list = PublicIPAddress.list( - self.apiclient, - account=self.account.name, - domainid=self.account.domainid, - listall=True, - issourcenat=True, - vpcid=vpc.id - ) + self.apiclient, + account=self.account.name, + domainid=self.account.domainid, + listall=True, + issourcenat=True, + vpcid=vpc.id + ) self.assertEqual(src_nat_list[0].ipaddress, - routers[0].publicip, - "Source Nat IP address was not allocated to VR" - ) + routers[0].publicip, + "Source Nat IP address was not allocated to VR" + ) @attr(tags=["advanced", "intervlan"], required_hardware="false") def test_10_nonoverlaping_cidrs(self): @@ -1594,43 +1616,43 @@ class TestVPC(cloudstackTestCase): self.services["vpc"]["cidr"] = "10.1.1.1/16" self.debug("Creating a VPC network in the account: %s" % - self.account.name) + self.account.name) vpc_1 = VPC.create( - self.apiclient, - self.services["vpc"], - vpcofferingid=self.vpc_off.id, - zoneid=self.zone.id, - account=self.account.name, - domainid=self.account.domainid - ) + self.apiclient, + self.services["vpc"], + vpcofferingid=self.vpc_off.id, + zoneid=self.zone.id, + account=self.account.name, + domainid=self.account.domainid + ) self.validate_vpc_network(vpc_1) self.services["vpc"]["cidr"] = "10.2.1.1/16" self.debug( - "Creating a non-overlapping VPC network in the account: %s" % - self.account.name) + "Creating a non-overlapping VPC network in the account: %s" % + self.account.name) vpc_2 = VPC.create( - self.apiclient, - self.services["vpc"], - vpcofferingid=self.vpc_off.id, - zoneid=self.zone.id, - account=self.account.name, - domainid=self.account.domainid - ) + self.apiclient, + self.services["vpc"], + vpcofferingid=self.vpc_off.id, + zoneid=self.zone.id, + account=self.account.name, + domainid=self.account.domainid + ) self.validate_vpc_network(vpc_2) self.services["vpc"]["cidr"] = "10.1.1.1/16" self.debug("Creating a overlapping VPC network in the account: %s" % - self.account.name) + self.account.name) try: vpc_3 = VPC.create( - self.apiclient, - self.services["vpc"], - vpcofferingid=self.vpc_off.id, - zoneid=self.zone.id, - account=self.account.name, - domainid=self.account.domainid - ) + self.apiclient, + self.services["vpc"], + vpcofferingid=self.vpc_off.id, + zoneid=self.zone.id, + account=self.account.name, + domainid=self.account.domainid + ) self.debug("%s" % vpc_3) except Exception as e: self.debug("%s" % e) @@ -1638,7 +1660,7 @@ class TestVPC(cloudstackTestCase): else: assert("VPC created with overlapping CIDR") return - + @attr(tags=["advanced", "intervlan"], required_hardware="true") def test_11_deploy_vm_wo_network_netdomain(self): """ Test deployment of vm in a VPC without network domain @@ -1648,7 +1670,7 @@ class TestVPC(cloudstackTestCase): # 2. Add network without networkDomain to this VPC. # 3. Deploy VM in this network. - if self.zone.domain == None: + if self.zone.domain is None: cmd = updateZone.updateZoneCmd() cmd.id = self.zone.id cmd.domain = "test.domain.org" @@ -1657,22 +1679,22 @@ class TestVPC(cloudstackTestCase): self.services["vpc"]["cidr"] = "10.1.1.1/16" self.debug("creating a VPC network in the account: %s" % - self.account.name) - + self.account.name) + vpc = VPC.create( - self.apiclient, - self.services["vpc"], - vpcofferingid=self.vpc_off.id, - zoneid=self.zone.id, - account=self.account.name, - domainid=self.account.domainid - ) + self.apiclient, + self.services["vpc"], + vpcofferingid=self.vpc_off.id, + zoneid=self.zone.id, + account=self.account.name, + domainid=self.account.domainid + ) self.validate_vpc_network(vpc) self.network_offering = NetworkOffering.create( - self.apiclient, - self.services["network_offering"], - conservemode=False - ) + self.apiclient, + self.services["network_offering"], + conservemode=False + ) # Enable Network offering self.network_offering.update(self.apiclient, state='Enabled') self.cleanup.append(self.network_offering) @@ -1684,98 +1706,102 @@ class TestVPC(cloudstackTestCase): # Creating network using the network offering created self.debug("Creating network with network offering: %s" % - self.network_offering.id) + self.network_offering.id) network = Network.create( - self.apiclient, - self.services["network"], - accountid=self.account.name, - domainid=self.account.domainid, - networkofferingid=self.network_offering.id, - zoneid=self.zone.id, - gateway=gateway, - vpcid=vpc.id, - ) + self.apiclient, + self.services["network"], + accountid=self.account.name, + domainid=self.account.domainid, + networkofferingid=self.network_offering.id, + zoneid=self.zone.id, + gateway=gateway, + vpcid=vpc.id, + ) self.debug("Created network with ID: %s" % network.id) # Spawn an instance in that network virtual_machine = VirtualMachine.create( - self.apiclient, - self.services["virtual_machine"], - accountid=self.account.name, - domainid=self.account.domainid, - serviceofferingid=self.service_offering.id, - networkids=[str(network.id)] - ) + self.apiclient, + self.services["virtual_machine"], + accountid=self.account.name, + domainid=self.account.domainid, + serviceofferingid=self.service_offering.id, + networkids=[str(network.id)] + ) self.debug("Deployed VM in network: %s" % network.id) - self.validate_vm_netdomain(virtual_machine, vpc, network, self.zone.domain) + self.validate_vm_netdomain( + virtual_machine, + vpc, + network, + self.zone.domain) def validate_vm_netdomain(self, vm, vpc, network, expected_netdomain): self.debug("Associating public IP for network: %s" % network.name) src_nat_ip_addr = PublicIPAddress.create( - self.apiclient, - zoneid=self.zone.id, - accountid=self.account.name, - domainid=self.account.domainid, - networkid=network.id, - vpcid=vpc.id - ) + self.apiclient, + zoneid=self.zone.id, + accountid=self.account.name, + domainid=self.account.domainid, + networkid=network.id, + vpcid=vpc.id + ) self.debug("Associated %s with network %s" % ( - src_nat_ip_addr.ipaddress.ipaddress, - network.id - )) + src_nat_ip_addr.ipaddress.ipaddress, + network.id + )) self.debug("Public IP %s" % src_nat_ip_addr.__dict__) # Create NAT rule nat_rule = NATRule.create( - self.apiclient, - vm, - self.services["natrule"], - src_nat_ip_addr.ipaddress.id, - openfirewall=False, - networkid=network.id, - vpcid=vpc.id - ) + self.apiclient, + vm, + self.services["natrule"], + src_nat_ip_addr.ipaddress.id, + openfirewall=False, + networkid=network.id, + vpcid=vpc.id + ) list_nat_rule_response = NATRule.list( - self.apiclient, - id=nat_rule.id - ) + self.apiclient, + id=nat_rule.id + ) self.assertEqual( - isinstance(list_nat_rule_response, list), - True, - "Check list response returns a valid list" - ) + isinstance(list_nat_rule_response, list), + True, + "Check list response returns a valid list" + ) self.assertNotEqual( - len(list_nat_rule_response), - 0, - "Check Port Forwarding Rule is created" - ) + len(list_nat_rule_response), + 0, + "Check Port Forwarding Rule is created" + ) self.assertEqual( - list_nat_rule_response[0].id, - nat_rule.id, - "Check Correct Port forwarding Rule is returned" - ) + list_nat_rule_response[0].id, + nat_rule.id, + "Check Correct Port forwarding Rule is returned" + ) self.debug("Adding NetworkACl rules to make NAT rule accessible") - nwacl_nat = NetworkACL.create( - self.apiclient, - networkid=network.id, - services=self.services["natrule"], - traffictype='Ingress' - ) + NetworkACL.create( + self.apiclient, + networkid=network.id, + services=self.services["natrule"], + traffictype='Ingress' + ) self.debug("SSHing into VM with IP address %s with NAT IP %s" % - ( - vm.ipaddress, - src_nat_ip_addr.ipaddress.ipaddress)) + ( + vm.ipaddress, + src_nat_ip_addr.ipaddress.ipaddress)) try: ssh_1 = vm.get_ssh_client( - ipaddress=src_nat_ip_addr.ipaddress.ipaddress) + ipaddress=src_nat_ip_addr.ipaddress.ipaddress) self.debug("SSH into VM is successfully") # Ping to outsite world @@ -1783,15 +1809,15 @@ class TestVPC(cloudstackTestCase): except Exception as e: self.fail("Failed to SSH into VM - %s, %s" % - (vm.ssh_ip, e)) + (vm.ssh_ip, e)) vm_domain = res[1].split(" ")[1] self.assertEqual( - vm_domain, - expected_netdomain, - "The network domain assigned to virtual machine " - "is %s expected domain was %s" % - (vm_domain, expected_netdomain) - ) + vm_domain, + expected_netdomain, + "The network domain assigned to virtual machine " + "is %s expected domain was %s" % + (vm_domain, expected_netdomain) + ) @attr(tags=["advanced", "intervlan"], required_hardware="false") def test_12_deploy_vm_with_netdomain(self): @@ -1804,22 +1830,22 @@ class TestVPC(cloudstackTestCase): self.services["vpc"]["cidr"] = "10.1.1.1/16" self.debug("creating a VPC network in the account: %s" % - self.account.name) + self.account.name) vpc = VPC.create( - self.apiclient, - self.services["vpc"], - vpcofferingid=self.vpc_off.id, - zoneid=self.zone.id, - account=self.account.name, - domainid=self.account.domainid - ) + self.apiclient, + self.services["vpc"], + vpcofferingid=self.vpc_off.id, + zoneid=self.zone.id, + account=self.account.name, + domainid=self.account.domainid + ) self.validate_vpc_network(vpc) self.network_offering = NetworkOffering.create( - self.apiclient, - self.services["network_offering"], - conservemode=False - ) + self.apiclient, + self.services["network_offering"], + conservemode=False + ) # Enable Network offering self.network_offering.update(self.apiclient, state='Enabled') self.cleanup.append(self.network_offering) @@ -1831,22 +1857,22 @@ class TestVPC(cloudstackTestCase): # Creating network using the network offering created self.debug("Creating network with network offering: %s" % - self.network_offering.id) + self.network_offering.id) # Creation of network with different network domain than the one # specified in VPC should fail. with self.assertRaises(Exception): Network.create( - self.apiclient, - self.services["network"], - accountid=self.account.name, - domainid=self.account.domainid, - networkofferingid=self.network_offering.id, - zoneid=self.zone.id, - gateway=gateway, - vpcid=vpc.id, - networkdomain='test.netdomain' - ) + self.apiclient, + self.services["network"], + accountid=self.account.name, + domainid=self.account.domainid, + networkofferingid=self.network_offering.id, + zoneid=self.zone.id, + gateway=gateway, + vpcid=vpc.id, + networkdomain='test.netdomain' + ) @attr(tags=["advanced", "intervlan"], required_hardware="true") def test_13_deploy_vm_with_vpc_netdomain(self): @@ -1859,23 +1885,23 @@ class TestVPC(cloudstackTestCase): self.services["vpc"]["cidr"] = "10.1.1.1/16" self.debug("creating a VPC network in the account: %s" % - self.account.name) + self.account.name) netdomain = "cl2.internal" vpc = VPC.create( - self.apiclient, - self.services["vpc"], - vpcofferingid=self.vpc_off.id, - zoneid=self.zone.id, - account=self.account.name, - domainid=self.account.domainid, - networkDomain=netdomain - ) + self.apiclient, + self.services["vpc"], + vpcofferingid=self.vpc_off.id, + zoneid=self.zone.id, + account=self.account.name, + domainid=self.account.domainid, + networkDomain=netdomain + ) self.validate_vpc_network(vpc) self.network_offering = NetworkOffering.create( - self.apiclient, - self.services["network_offering"], - conservemode=False - ) + self.apiclient, + self.services["network_offering"], + conservemode=False + ) # Enable Network offering self.network_offering.update(self.apiclient, state='Enabled') self.cleanup.append(self.network_offering) @@ -1887,35 +1913,36 @@ class TestVPC(cloudstackTestCase): # Creating network using the network offering created self.debug("Creating network with network offering: %s" % - self.network_offering.id) + self.network_offering.id) network = Network.create( - self.apiclient, - self.services["network"], - accountid=self.account.name, - domainid=self.account.domainid, - networkofferingid=self.network_offering.id, - zoneid=self.zone.id, - gateway=gateway, - vpcid=vpc.id, - ) + self.apiclient, + self.services["network"], + accountid=self.account.name, + domainid=self.account.domainid, + networkofferingid=self.network_offering.id, + zoneid=self.zone.id, + gateway=gateway, + vpcid=vpc.id, + ) self.debug("Created network with ID: %s" % network.id) # Spawn an instance in that network virtual_machine = VirtualMachine.create( - self.apiclient, - self.services["virtual_machine"], - accountid=self.account.name, - domainid=self.account.domainid, - serviceofferingid=self.service_offering.id, - networkids=[str(network.id)] - ) + self.apiclient, + self.services["virtual_machine"], + accountid=self.account.name, + domainid=self.account.domainid, + serviceofferingid=self.service_offering.id, + networkids=[str(network.id)] + ) self.debug("Deployed VM in network: %s" % network.id) self.validate_vm_netdomain(virtual_machine, vpc, network, netdomain) @attr(tags=["advanced", "intervlan"], required_hardware="false") def test_14_deploy_vm_1(self): - """ Test vm deploy in network by a user where VPC was created without account/domain ID + """ Test vm deploy in network by a user where VPC was created + without account/domain ID """ # 1. Create VPC without providing account/domain ID. @@ -1923,35 +1950,34 @@ class TestVPC(cloudstackTestCase): # 3. Deploy VM in this network user = Account.create( - self.apiclient, - self.services["account"] - ) + self.apiclient, + self.services["account"] + ) self.debug("Created account: %s" % user.name) self.cleanup.append(user) self.services["vpc"]["cidr"] = "10.1.1.1/16" self.debug("creating a VPC network in the account: %s" % - user.name) + user.name) userapiclient = self.testClient.getUserApiClient( - UserName=user.name, - DomainName=user.domain, - type=0) - + UserName=user.name, + DomainName=user.domain, + type=0) vpc = VPC.create( - userapiclient, - self.services["vpc"], - vpcofferingid=self.vpc_off.id, - zoneid=self.zone.id, - ) + userapiclient, + self.services["vpc"], + vpcofferingid=self.vpc_off.id, + zoneid=self.zone.id, + ) self.validate_vpc_network(vpc) self.network_offering = NetworkOffering.create( - self.apiclient, - self.services["network_offering"], - conservemode=False - ) + self.apiclient, + self.services["network_offering"], + conservemode=False + ) # Enable Network offering self.network_offering.update(self.apiclient, state='Enabled') self.cleanup.append(self.network_offering) @@ -1963,76 +1989,72 @@ class TestVPC(cloudstackTestCase): # Creating network using the network offering created self.debug("Creating network with network offering: %s" % - self.network_offering.id) + self.network_offering.id) network = Network.create( - userapiclient, - self.services["network"], - networkofferingid=self.network_offering.id, - zoneid=self.zone.id, - gateway=gateway, - vpcid=vpc.id - ) + userapiclient, + self.services["network"], + networkofferingid=self.network_offering.id, + zoneid=self.zone.id, + gateway=gateway, + vpcid=vpc.id + ) self.debug("Created network with ID: %s" % network.id) # Spawn an instance in that network virtual_machine = VirtualMachine.create( - userapiclient, - self.services["virtual_machine"], - serviceofferingid=self.service_offering.id, - networkids=[str(network.id)] - ) + userapiclient, + self.services["virtual_machine"], + serviceofferingid=self.service_offering.id, + networkids=[str(network.id)] + ) self.debug("Deployed VM in network: %s" % network.id) self.assertNotEqual(virtual_machine, - None, - "VM creation in the network failed") + None, + "VM creation in the network failed") return @attr(tags=["advanced", "intervlan"], required_hardware="false") def test_15_deploy_vm_2(self): - """ Test deployment of vm in a network in a domain admin account where VPC is created without account/domain ID + """ Test deployment of vm in a network in a domain admin + account where VPC is created without account/domain ID """ # 1. Create VPC without providing account/domain ID. # 2. Add network with using domain admin account to this VPC. # 3. Deploy VM in this network - domain = Domain.create( - self.api_client, - self.services["domain"], - ) - user = Account.create( - self.apiclient, - self.services["account"] - ) + self.apiclient, + self.services["account"] + ) self.debug("Created account: %s" % user.name) self.cleanup.append(user) self.services["vpc"]["cidr"] = "10.1.1.1/16" self.debug("creating a VPC network in the account: %s" % - user.name) + user.name) - #0 - User, 1 - Root Admin, 2 - Domain Admin + # 0 - User, 1 - Root Admin, 2 - Domain Admin userapiclient = self.testClient.getUserApiClient( - UserName=user.name, - DomainName=self.services["domain"]["name"], - type=2) + UserName=user.name, + DomainName=self.services["domain"]["name"], + type=2) vpc = VPC.create( - userapiclient, - self.services["vpc"], - vpcofferingid=self.vpc_off.id, - zoneid=self.zone.id, - ) + userapiclient, + self.services["vpc"], + vpcofferingid=self.vpc_off.id, + zoneid=self.zone.id, + ) self.validate_vpc_network(vpc) self.network_offering = NetworkOffering.create( - self.apiclient, - self.services["network_offering"], - conservemode=False - ) + self.apiclient, + self.services["network_offering"], + conservemode=False + ) # Enable Network offering self.network_offering.update(self.apiclient, state='Enabled') self.cleanup.append(self.network_offering) @@ -2044,29 +2066,29 @@ class TestVPC(cloudstackTestCase): # Creating network using the network offering created self.debug("Creating network with network offering: %s" % - self.network_offering.id) + self.network_offering.id) network = Network.create( - userapiclient, - self.services["network"], - networkofferingid=self.network_offering.id, - zoneid=self.zone.id, - gateway=gateway, - vpcid=vpc.id - ) + userapiclient, + self.services["network"], + networkofferingid=self.network_offering.id, + zoneid=self.zone.id, + gateway=gateway, + vpcid=vpc.id + ) self.debug("Created network with ID: %s" % network.id) # Spawn an instance in that network virtual_machine = VirtualMachine.create( - userapiclient, - self.services["virtual_machine"], - serviceofferingid=self.service_offering.id, - networkids=[str(network.id)] - ) + userapiclient, + self.services["virtual_machine"], + serviceofferingid=self.service_offering.id, + networkids=[str(network.id)] + ) self.debug("Deployed VM in network: %s" % network.id) self.assertNotEqual(virtual_machine, - None, - "VM creation in the network failed") + None, + "VM creation in the network failed") return @@ -2075,41 +2097,43 @@ class TestVPC(cloudstackTestCase): """ Test deployment of vm in a network by root admin for user. """ - #1. As root admin account , - # Create VPC(name,zoneId,cidr,vpcOfferingId,networkDomain by passing user Account/domain ID. - #2. As the user account used in step1 , create a network as part of this VPC. - #3. Deploy Vms as part of this network. + # 1. As root admin account , + # Create VPC(name,zoneId,cidr,vpcOfferingId,networkDomain by + # passing user Account/domain ID. + # 2. As the user account used in step1 , create a network as part + # of this VPC. + # 3. Deploy Vms as part of this network. user = Account.create( - self.apiclient, - self.services["account"] - ) + self.apiclient, + self.services["account"] + ) self.debug("Created account: %s" % user.name) self.cleanup.append(user) self.services["vpc"]["cidr"] = "10.1.1.1/16" self.debug("creating a VPC network in the account: %s" % - user.name) + user.name) userapiclient = self.testClient.getUserApiClient( - UserName=user.name, - DomainName=user.domain, - type=0) + UserName=user.name, + DomainName=user.domain, + type=0) vpc = VPC.create( - self.apiclient, - self.services["vpc"], - account=user.name, - domainid=user.domainid, - vpcofferingid=self.vpc_off.id, - zoneid=self.zone.id, - ) + self.apiclient, + self.services["vpc"], + account=user.name, + domainid=user.domainid, + vpcofferingid=self.vpc_off.id, + zoneid=self.zone.id, + ) self.validate_vpc_network(vpc) self.network_offering = NetworkOffering.create( - self.apiclient, - self.services["network_offering"], - conservemode=False - ) + self.apiclient, + self.services["network_offering"], + conservemode=False + ) # Enable Network offering self.network_offering.update(self.apiclient, state='Enabled') self.cleanup.append(self.network_offering) @@ -2121,29 +2145,29 @@ class TestVPC(cloudstackTestCase): # Creating network using the network offering created self.debug("Creating network with network offering: %s" % - self.network_offering.id) + self.network_offering.id) network = Network.create( - userapiclient, - self.services["network"], - networkofferingid=self.network_offering.id, - zoneid=self.zone.id, - gateway=gateway, - vpcid=vpc.id - ) + userapiclient, + self.services["network"], + networkofferingid=self.network_offering.id, + zoneid=self.zone.id, + gateway=gateway, + vpcid=vpc.id + ) self.debug("Created network with ID: %s" % network.id) # Spawn an instance in that network virtual_machine = VirtualMachine.create( - userapiclient, - self.services["virtual_machine"], - serviceofferingid=self.service_offering.id, - networkids=[str(network.id)] - ) + userapiclient, + self.services["virtual_machine"], + serviceofferingid=self.service_offering.id, + networkids=[str(network.id)] + ) self.debug("Deployed VM in network: %s" % network.id) self.assertNotEqual(virtual_machine, - None, - "VM creation in the network failed") + None, + "VM creation in the network failed") return @@ -2152,109 +2176,102 @@ class TestVPC(cloudstackTestCase): """ Test deployment of vm in a network by domain admin for user. """ - #1. As domain admin account , Create + # 1. As domain admin account , Create # VPC(name,zoneId,cidr,vpcOfferingId,networkDomain # by passing user Account/domain ID. - #2. As the user account used in step1, create network as part of this VPC - #3. Deploy Vms as part of this network. - - domain = Domain.create( - self.api_client, - self.services["domain"], - ) + # 2. As the user account used in step1, create network as part of + # this VPC + # 3. Deploy Vms as part of this network. domain_admin = Account.create( - self.apiclient, - self.services["domain_admin"] - ) + self.apiclient, + self.services["domain_admin"] + ) self.debug("Created account: %s" % domain_admin.name) self.cleanup.append(domain_admin) da_apiclient = self.testClient.getUserApiClient( - UserName=domain_admin.name, - DomainName=domain_admin.domain, - type=2) + UserName=domain_admin.name, + DomainName=domain_admin.domain, + type=2) user = Account.create( - self.apiclient, - self.services["account"] - ) + self.apiclient, + self.services["account"] + ) self.debug("Created account: %s" % user.name) self.cleanup.append(user) self.services["vpc"]["cidr"] = "10.1.1.1/16" self.debug("creating a VPC network in the account: %s" % - user.name) + user.name) - #0 - User, 1 - Root Admin, 2 - Domain Admin - userapiclient = self.testClient.getUserApiClient( - UserName=user.name, - DomainName=user.domain, - type=0) + # 0 - User, 1 - Root Admin, 2 - Domain Admin + self.testClient.getUserApiClient( + UserName=user.name, + DomainName=user.domain, + type=0) with self.assertRaises(CloudstackAPIException): - vpc = VPC.create( - da_apiclient, - self.services["vpc"], - account=user.name, - domainid=user.domainid, - vpcofferingid=self.vpc_off.id, - zoneid=self.zone.id, - ) + VPC.create( + da_apiclient, + self.services["vpc"], + account=user.name, + domainid=user.domainid, + vpcofferingid=self.vpc_off.id, + zoneid=self.zone.id, + ) @attr(tags=["advanced", "intervlan"], required_hardware="false") def test_18_create_net_for_user_diff_domain_by_doadmin(self): """ Test creation of network by domain admin for user from different domain """ - #1. As domain admin account , Create VPC(name,zoneId,cidr,vpcOfferingId,networkDomain) without passing Account/domain ID. - #2. As any User account that is not under this domain , create a network as part of this VPC. - - domain = Domain.create( - self.api_client, - self.services["domain"], - ) + # 1. As domain admin account , Create VPC(name,zoneId,cidr, + # vpcOfferingId,networkDomain) without passing Account/domain ID. + # 2. As any User account that is not under this domain , create a + # network as part of this VPC. domain_admin = Account.create( - self.apiclient, - self.services["domain_admin"] - ) + self.apiclient, + self.services["domain_admin"] + ) self.debug("Created account: %s" % domain_admin.name) self.cleanup.append(domain_admin) da_apiclient = self.testClient.getUserApiClient( - UserName=domain_admin.name, - DomainName=self.services["domain"]["name"], - type=2) + UserName=domain_admin.name, + DomainName=self.services["domain"]["name"], + type=2) user = Account.create( - self.apiclient, - self.services["account"] - ) + self.apiclient, + self.services["account"] + ) self.debug("Created account: %s" % user.name) self.cleanup.append(user) self.services["vpc"]["cidr"] = "10.1.1.1/16" self.debug("creating a VPC network in the account: %s" % - user.name) + user.name) - #0 - User, 1 - Root Admin, 2 - Domain Admin + # 0 - User, 1 - Root Admin, 2 - Domain Admin userapiclient = self.testClient.getUserApiClient( - UserName=user.name, - DomainName=user.domain, - type=0) + UserName=user.name, + DomainName=user.domain, + type=0) vpc = VPC.create( - da_apiclient, - self.services["vpc"], - vpcofferingid=self.vpc_off.id, - zoneid=self.zone.id, - ) + da_apiclient, + self.services["vpc"], + vpcofferingid=self.vpc_off.id, + zoneid=self.zone.id, + ) self.validate_vpc_network(vpc) self.network_offering = NetworkOffering.create( - self.apiclient, - self.services["network_offering"], - conservemode=False - ) + self.apiclient, + self.services["network_offering"], + conservemode=False + ) # Enable Network offering self.network_offering.update(self.apiclient, state='Enabled') self.cleanup.append(self.network_offering) @@ -2266,17 +2283,17 @@ class TestVPC(cloudstackTestCase): # Creating network using the network offering created self.debug("Creating network with network offering: %s" % - self.network_offering.id) + self.network_offering.id) with self.assertRaises(Exception): - network = Network.create( - userapiclient, - self.services["network"], - networkofferingid=self.network_offering.id, - zoneid=self.zone.id, - gateway=gateway, - vpcid=vpc.id - ) + Network.create( + userapiclient, + self.services["network"], + networkofferingid=self.network_offering.id, + zoneid=self.zone.id, + gateway=gateway, + vpcid=vpc.id + ) @attr(tags=["advanced", "intervlan"], required_hardware="false") def test_19_create_vpc_wo_params(self): @@ -2289,54 +2306,54 @@ class TestVPC(cloudstackTestCase): self.services["vpc"]["cidr"] = "10.1.1.1/16" self.debug("creating a VPC network in the account: %s" % - self.account.name) + self.account.name) # Create VPC without vpcOffering param with self.assertRaises(Exception): - vpc = VPC.create( - self.apiclient, - self.services["vpc"], - zoneid=self.zone.id, - account=self.account.name, - domainid=self.account.domainid - ) + VPC.create( + self.apiclient, + self.services["vpc"], + zoneid=self.zone.id, + account=self.account.name, + domainid=self.account.domainid + ) self.services["vpc_no_name"]["cidr"] = "10.1.1.1/16" # Create VPC without name param with self.assertRaises(Exception): - vpc = VPC.create( - self.apiclient, - self.services["vpc_no_name"], - vpcofferingid=self.vpc_off.id, - zoneid=self.zone.id, - account=self.account.name, - domainid=self.account.domainid - ) + VPC.create( + self.apiclient, + self.services["vpc_no_name"], + vpcofferingid=self.vpc_off.id, + zoneid=self.zone.id, + account=self.account.name, + domainid=self.account.domainid + ) # Create VPC without zoneid param with self.assertRaises(Exception): - vpc = VPC.create( - self.apiclient, - self.services["vpc"], - vpcofferingid=self.vpc_off.id, - account=self.account.name, - domainid=self.account.domainid - ) + VPC.create( + self.apiclient, + self.services["vpc"], + vpcofferingid=self.vpc_off.id, + account=self.account.name, + domainid=self.account.domainid + ) vpc_wo_cidr = {"name": "TestVPC_WO_CIDR", - "displaytext": "TestVPC_WO_CIDR" - } + "displaytext": "TestVPC_WO_CIDR" + } # Create VPC without CIDR with self.assertRaises(Exception): - vpc = VPC.create( - self.apiclient, - vpc_wo_cidr, - vpcofferingid=self.vpc_off.id, - zoneid=self.zone.id, - account=self.account.name, - domainid=self.account.domainid - ) + VPC.create( + self.apiclient, + vpc_wo_cidr, + vpcofferingid=self.vpc_off.id, + zoneid=self.zone.id, + account=self.account.name, + domainid=self.account.domainid + ) @attr(tags=["advanced", "intervlan"], required_hardware="false") def test_20_update_vpc_name_display_text(self): @@ -2350,22 +2367,22 @@ class TestVPC(cloudstackTestCase): self.services["vpc"]["cidr"] = "10.1.1.1/16" self.debug("creating a VPC network in the account: %s" % - self.account.name) + self.account.name) vpc = VPC.create( - self.apiclient, - self.services["vpc"], - vpcofferingid=self.vpc_off.id, - zoneid=self.zone.id, - account=self.account.name, - domainid=self.account.domainid - ) + self.apiclient, + self.services["vpc"], + vpcofferingid=self.vpc_off.id, + zoneid=self.zone.id, + account=self.account.name, + domainid=self.account.domainid + ) self.validate_vpc_network(vpc) self.network_offering = NetworkOffering.create( - self.apiclient, - self.services["network_offering"], - conservemode=False - ) + self.apiclient, + self.services["network_offering"], + conservemode=False + ) # Enable Network offering self.network_offering.update(self.apiclient, state='Enabled') self.cleanup.append(self.network_offering) @@ -2377,41 +2394,41 @@ class TestVPC(cloudstackTestCase): # Creating network using the network offering created self.debug("Creating network with network offering: %s" % - self.network_offering.id) + self.network_offering.id) network = Network.create( - self.apiclient, - self.services["network"], - accountid=self.account.name, - domainid=self.account.domainid, - networkofferingid=self.network_offering.id, - zoneid=self.zone.id, - gateway=gateway, - vpcid=vpc.id - ) + self.apiclient, + self.services["network"], + accountid=self.account.name, + domainid=self.account.domainid, + networkofferingid=self.network_offering.id, + zoneid=self.zone.id, + gateway=gateway, + vpcid=vpc.id + ) self.debug("Created network with ID: %s" % network.id) new_name = "New VPC" new_display_text = "New display text" vpc.update( - self.apiclient, - name=new_name, - displaytext=new_display_text - ) + self.apiclient, + name=new_name, + displaytext=new_display_text + ) vpc_networks = VPC.list( - self.apiclient, - id=vpc.id - ) - + self.apiclient, + id=vpc.id + ) + self.assertEqual( - isinstance(vpc_networks, list), - True, - "List VPC network should return a valid list" - ) + isinstance(vpc_networks, list), + True, + "List VPC network should return a valid list" + ) self.assertEqual(vpc_networks[0].name, - new_name, - "Updation of VPC name failed.") + new_name, + "Updation of VPC name failed.") self.assertEqual(vpc_networks[0].displaytext, - new_display_text, - "Updation of VPC display text failed.") + new_display_text, + "Updation of VPC display text failed.") From 3736d9435ad7e26f889dbb3b90fa7ce457ea4967 Mon Sep 17 00:00:00 2001 From: Sangeetha Hariharan Date: Tue, 21 Oct 2014 18:12:03 -0700 Subject: [PATCH 015/391] CLOUDSTACK-7762 -[Automation] - Fix test failure for test_02_revert_vm_snapshots in smoke/test_vm_snapshots.py --- test/integration/smoke/test_vm_snapshots.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/integration/smoke/test_vm_snapshots.py b/test/integration/smoke/test_vm_snapshots.py index 131da998508..ef1353a6df9 100644 --- a/test/integration/smoke/test_vm_snapshots.py +++ b/test/integration/smoke/test_vm_snapshots.py @@ -202,6 +202,9 @@ class TestVmSnapshot(cloudstackTestCase): "Check the snapshot of vm is ready!" ) + # Stop Virtual machine befor reverting VM to a snapshot taken without memory + self.virtual_machine.stop(self.apiclient) + VmSnapshot.revertToSnapshot(self.apiclient, list_snapshot_response[0].id) list_vm_response = list_virtual_machines( From e0c495d35754f167afa9779fb9a42ac16260e446 Mon Sep 17 00:00:00 2001 From: Koushik Das Date: Tue, 23 Dec 2014 11:56:40 +0530 Subject: [PATCH 016/391] Revert "CLOUDSTACK-7872: network getting shutdown inspite of running VM's in the network" This reverts commit 0bd34d389f93b56e74a6bb04054c8e419bf8571e. Network GC is broken with out-of-band VM movements due to the original commit, so reverting. --- .../com/cloud/network/dao/NetworkDaoImpl.java | 2 ++ .../schema/src/com/cloud/vm/dao/NicDao.java | 2 -- .../src/com/cloud/vm/dao/NicDaoImpl.java | 22 ------------------- .../com/cloud/network/NetworkModelImpl.java | 7 ------ 4 files changed, 2 insertions(+), 31 deletions(-) diff --git a/engine/schema/src/com/cloud/network/dao/NetworkDaoImpl.java b/engine/schema/src/com/cloud/network/dao/NetworkDaoImpl.java index 4a07455832a..0c556c843d9 100644 --- a/engine/schema/src/com/cloud/network/dao/NetworkDaoImpl.java +++ b/engine/schema/src/com/cloud/network/dao/NetworkDaoImpl.java @@ -244,6 +244,7 @@ public class NetworkDaoImpl extends GenericDaoBase implements N GarbageCollectedSearch = createSearchBuilder(Long.class); GarbageCollectedSearch.selectFields(GarbageCollectedSearch.entity().getId()); SearchBuilder join7 = _ntwkOpDao.createSearchBuilder(); + join7.and("activenics", join7.entity().getActiveNicsCount(), Op.EQ); join7.and("gc", join7.entity().isGarbageCollected(), Op.EQ); join7.and("check", join7.entity().isCheckForGc(), Op.EQ); GarbageCollectedSearch.join("ntwkOpGC", join7, GarbageCollectedSearch.entity().getId(), join7.entity().getId(), JoinBuilder.JoinType.INNER); @@ -437,6 +438,7 @@ public class NetworkDaoImpl extends GenericDaoBase implements N public List findNetworksToGarbageCollect() { SearchCriteria sc = GarbageCollectedSearch.create(); sc.setJoinParameters("ntwkOffGC", "isPersistent", false); + sc.setJoinParameters("ntwkOpGC", "activenics", 0); sc.setJoinParameters("ntwkOpGC", "gc", true); sc.setJoinParameters("ntwkOpGC", "check", true); return customSearch(sc, null); diff --git a/engine/schema/src/com/cloud/vm/dao/NicDao.java b/engine/schema/src/com/cloud/vm/dao/NicDao.java index 9f153f8eeb4..a7ad01692cf 100644 --- a/engine/schema/src/com/cloud/vm/dao/NicDao.java +++ b/engine/schema/src/com/cloud/vm/dao/NicDao.java @@ -74,6 +74,4 @@ public interface NicDao extends GenericDao { List listByNetworkIdTypeAndGatewayAndBroadcastUri(long networkId, VirtualMachine.Type vmType, String gateway, URI broadcastUri); int countNicsForStartingVms(long networkId); - - int countNicsForRunningVms(long networkId); } diff --git a/engine/schema/src/com/cloud/vm/dao/NicDaoImpl.java b/engine/schema/src/com/cloud/vm/dao/NicDaoImpl.java index ca44b631e4f..2a9a6025071 100644 --- a/engine/schema/src/com/cloud/vm/dao/NicDaoImpl.java +++ b/engine/schema/src/com/cloud/vm/dao/NicDaoImpl.java @@ -46,7 +46,6 @@ public class NicDaoImpl extends GenericDaoBase implements NicDao { private SearchBuilder NonReleasedSearch; private GenericSearchBuilder CountBy; private GenericSearchBuilder CountByForStartingVms; - private GenericSearchBuilder CountByForRunningVms; @Inject VMInstanceDao _vmDao; @@ -96,17 +95,6 @@ public class NicDaoImpl extends GenericDaoBase implements NicDao { join1.and("state", join1.entity().getState(), Op.EQ); CountByForStartingVms.join("vm", join1, CountByForStartingVms.entity().getInstanceId(), join1.entity().getId(), JoinBuilder.JoinType.INNER); CountByForStartingVms.done(); - - CountByForRunningVms = createSearchBuilder(Integer.class); - CountByForRunningVms.select(null, Func.COUNT, CountByForRunningVms.entity().getId()); - CountByForRunningVms.and("networkId", CountByForRunningVms.entity().getNetworkId(), Op.EQ); - CountByForRunningVms.and("removed", CountByForRunningVms.entity().getRemoved(), Op.NULL); - SearchBuilder join2 = _vmDao.createSearchBuilder(); - join2.and("state", join2.entity().getState(), Op.EQ); - join2.and("type", join2.entity().getType(), Op.EQ); - CountByForRunningVms.join("vm", join2, CountByForRunningVms.entity().getInstanceId(), join2.entity().getId(), JoinBuilder.JoinType.INNER); - CountByForRunningVms.done(); - } @Override @@ -303,14 +291,4 @@ public class NicDaoImpl extends GenericDaoBase implements NicDao { List results = customSearch(sc, null); return results.get(0); } - - @Override - public int countNicsForRunningVms(long networkId) { - SearchCriteria sc = CountByForRunningVms.create(); - sc.setParameters("networkId", networkId); - sc.setJoinParameters("vm", "state", VirtualMachine.State.Running); - sc.setJoinParameters("vm", "type", VirtualMachine.Type.User); - List results = customSearch(sc, null); - return results.get(0); - } } diff --git a/server/src/com/cloud/network/NetworkModelImpl.java b/server/src/com/cloud/network/NetworkModelImpl.java index 1fd354df6d3..60882124fcc 100755 --- a/server/src/com/cloud/network/NetworkModelImpl.java +++ b/server/src/com/cloud/network/NetworkModelImpl.java @@ -2232,13 +2232,6 @@ public class NetworkModelImpl extends ManagerBase implements NetworkModel { return false; } - // Due to VMSync issue, there can be cases where nic count is zero, but there can be VM's running in the network - // so add extra guard to check if network GC is actially required. - if (_nicDao.countNicsForRunningVms(networkId) > 0) { - s_logger.debug("Network id=" + networkId + " is not ready for GC as it has vms that are Running at the moment"); - return false; - } - return true; } From ab3af7b014d7391da056e6461e1c1f432e62d827 Mon Sep 17 00:00:00 2001 From: Chandan Purushothama Date: Wed, 22 Oct 2014 12:09:52 -0700 Subject: [PATCH 017/391] CLOUDSTACK-7769 - Fixed test_ssvm.py script --- test/integration/smoke/test_ssvm.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/test/integration/smoke/test_ssvm.py b/test/integration/smoke/test_ssvm.py index 5713569e4ff..ed9eab08c2d 100644 --- a/test/integration/smoke/test_ssvm.py +++ b/test/integration/smoke/test_ssvm.py @@ -157,8 +157,16 @@ class TestSSVMs(cloudstackTestCase): "Check list response returns a valid list" ) iprange = ipranges_response[0] - - self.assertEqual( + + #Fetch corresponding Physical Network of SSVM's Zone + listphyntwk = PhysicalNetwork.list( + self.apiclient, + zoneid=ssvm.zoneid + ) + + # Execute the following assertion in all zones except EIP-ELB Zones + if not (self.zone.networktype.lower() == 'basic' and isinstance(NetScaler.list(self.apiclient,physicalnetworkid=listphyntwk[0].id), list) is True): + self.assertEqual( ssvm.gateway, iprange.gateway, "Check gateway with that of corresponding ip range" @@ -274,7 +282,15 @@ class TestSSVMs(cloudstackTestCase): ) iprange = ipranges_response[0] - self.assertEqual( + #Fetch corresponding Physical Network of SSVM's Zone + listphyntwk = PhysicalNetwork.list( + self.apiclient, + zoneid=cpvm.zoneid + ) + + # Execute the following assertion in all zones except EIP-ELB Zones + if not (self.zone.networktype.lower() == 'basic' and isinstance(NetScaler.list(self.apiclient,physicalnetworkid=listphyntwk[0].id), list) is True): + self.assertEqual( cpvm.gateway, iprange.gateway, "Check gateway with that of corresponding ip range" From 1413efcac3c80e0dd6561a35cf8fa2df10c2ad46 Mon Sep 17 00:00:00 2001 From: Gaurav Aradhye Date: Mon, 22 Dec 2014 14:49:25 +0530 Subject: [PATCH 018/391] CLOUDSTACK-8096: Fixed test_ssvm.py for issues while checking the result of diagnostic scripts Signed-off-by: SrikanteswaraRao Talluri --- test/integration/smoke/test_ssvm.py | 1068 +++++++++++++---------- tools/marvin/marvin/config/test_data.py | 3 +- 2 files changed, 595 insertions(+), 476 deletions(-) diff --git a/test/integration/smoke/test_ssvm.py b/test/integration/smoke/test_ssvm.py index ed9eab08c2d..0df0302e029 100644 --- a/test/integration/smoke/test_ssvm.py +++ b/test/integration/smoke/test_ssvm.py @@ -5,9 +5,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -16,21 +16,29 @@ # under the License. """ BVT tests for SSVM """ -#Import Local Modules -import marvin -from marvin.cloudstackTestCase import * -from marvin.cloudstackAPI import * -from marvin.sshClient import SshClient -from marvin.lib.utils import * -from marvin.lib.base import * -from marvin.lib.common import * +# Import Local Modules +from marvin.cloudstackTestCase import cloudstackTestCase +from marvin.cloudstackAPI import (stopSystemVm, + rebootSystemVm, + destroySystemVm) +from marvin.lib.utils import (cleanup_resources, + get_process_status, + get_host_credentials) +from marvin.lib.base import (PhysicalNetwork, + NetScaler) +from marvin.lib.common import (get_zone, + list_hosts, + list_ssvms, + list_zones, + list_vlan_ipranges) from nose.plugins.attrib import attr import telnetlib -#Import System modules +# Import System modules import time _multiprocess_shared_ = True + class TestSSVMs(cloudstackTestCase): def setUp(self): @@ -43,7 +51,7 @@ class TestSSVMs(cloudstackTestCase): def tearDown(self): try: - #Clean up, terminate the created templates + # Clean up, terminate the created templates cleanup_resources(self.apiclient, self.cleanup) except Exception as e: @@ -55,9 +63,9 @@ class TestSSVMs(cloudstackTestCase): while True: list_host_response = list_hosts( - self.apiclient, - name=vmname - ) + self.apiclient, + name=vmname + ) if list_host_response and list_host_response[0].state == 'Up': break @@ -68,8 +76,14 @@ class TestSSVMs(cloudstackTestCase): time.sleep(self.services["sleep"]) timeout = timeout - 1 - - @attr(tags = ["advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") + @attr( + tags=[ + "advanced", + "advancedns", + "smoke", + "basic", + "sg"], + required_hardware="false") def test_01_list_sec_storage_vm(self): """Test List secondary storage VMs """ @@ -85,117 +99,129 @@ class TestSSVMs(cloudstackTestCase): # 5. DNS entries must match those given for the zone list_ssvm_response = list_ssvms( - self.apiclient, - systemvmtype='secondarystoragevm', - state='Running', - ) + self.apiclient, + systemvmtype='secondarystoragevm', + state='Running', + ) self.assertEqual( - isinstance(list_ssvm_response, list), - True, - "Check list response returns a valid list" - ) - #Verify SSVM response + isinstance(list_ssvm_response, list), + True, + "Check list response returns a valid list" + ) + # Verify SSVM response self.assertNotEqual( - len(list_ssvm_response), - 0, - "Check list System VMs response" - ) + len(list_ssvm_response), + 0, + "Check list System VMs response" + ) list_zones_response = list_zones(self.apiclient) - + self.assertEqual( - isinstance(list_zones_response, list), - True, - "Check list response returns a valid list" - ) + isinstance(list_zones_response, list), + True, + "Check list response returns a valid list" + ) self.debug("Number of zones: %s" % len(list_zones_response)) self.debug("Number of SSVMs: %s" % len(list_ssvm_response)) # Number of Sec storage VMs = No of Zones self.assertEqual( - len(list_ssvm_response), - len(list_zones_response), - "Check number of SSVMs with number of zones" - ) - #For each secondary storage VM check private IP, - #public IP, link local IP and DNS + len(list_ssvm_response), + len(list_zones_response), + "Check number of SSVMs with number of zones" + ) + # For each secondary storage VM check private IP, + # public IP, link local IP and DNS for ssvm in list_ssvm_response: self.debug("SSVM state: %s" % ssvm.state) self.assertEqual( - ssvm.state, - 'Running', - "Check whether state of SSVM is running" - ) + ssvm.state, + 'Running', + "Check whether state of SSVM is running" + ) self.assertEqual( - hasattr(ssvm, 'privateip'), - True, - "Check whether SSVM has private IP field" - ) + hasattr(ssvm, 'privateip'), + True, + "Check whether SSVM has private IP field" + ) self.assertEqual( - hasattr(ssvm, 'linklocalip'), - True, - "Check whether SSVM has link local IP field" - ) + hasattr(ssvm, 'linklocalip'), + True, + "Check whether SSVM has link local IP field" + ) self.assertEqual( - hasattr(ssvm, 'publicip'), - True, - "Check whether SSVM has public IP field" - ) + hasattr(ssvm, 'publicip'), + True, + "Check whether SSVM has public IP field" + ) - #Fetch corresponding ip ranges information from listVlanIpRanges + # Fetch corresponding ip ranges information from listVlanIpRanges ipranges_response = list_vlan_ipranges( - self.apiclient, - zoneid=ssvm.zoneid - ) + self.apiclient, + zoneid=ssvm.zoneid + ) self.assertEqual( - isinstance(ipranges_response, list), - True, - "Check list response returns a valid list" - ) + isinstance(ipranges_response, list), + True, + "Check list response returns a valid list" + ) iprange = ipranges_response[0] - - #Fetch corresponding Physical Network of SSVM's Zone + + # Fetch corresponding Physical Network of SSVM's Zone listphyntwk = PhysicalNetwork.list( - self.apiclient, - zoneid=ssvm.zoneid - ) - + self.apiclient, + zoneid=ssvm.zoneid + ) + # Execute the following assertion in all zones except EIP-ELB Zones - if not (self.zone.networktype.lower() == 'basic' and isinstance(NetScaler.list(self.apiclient,physicalnetworkid=listphyntwk[0].id), list) is True): + if not ( + self.zone.networktype.lower() == 'basic' and isinstance( + NetScaler.list( + self.apiclient, + physicalnetworkid=listphyntwk[0].id), + list) is True): self.assertEqual( - ssvm.gateway, - iprange.gateway, - "Check gateway with that of corresponding ip range" - ) + ssvm.gateway, + iprange.gateway, + "Check gateway with that of corresponding ip range" + ) - #Fetch corresponding zone information from listZones + # Fetch corresponding zone information from listZones zone_response = list_zones( - self.apiclient, - id=ssvm.zoneid - ) + self.apiclient, + id=ssvm.zoneid + ) self.assertEqual( - isinstance(zone_response, list), - True, - "Check list response returns a valid list" - ) + isinstance(zone_response, list), + True, + "Check list response returns a valid list" + ) self.assertEqual( - ssvm.dns1, - zone_response[0].dns1, - "Check DNS1 with that of corresponding zone" - ) + ssvm.dns1, + zone_response[0].dns1, + "Check DNS1 with that of corresponding zone" + ) self.assertEqual( - ssvm.dns2, - zone_response[0].dns2, - "Check DNS2 with that of corresponding zone" - ) + ssvm.dns2, + zone_response[0].dns2, + "Check DNS2 with that of corresponding zone" + ) return - @attr(tags = ["advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="false") + @attr( + tags=[ + "advanced", + "advancedns", + "smoke", + "basic", + "sg"], + required_hardware="false") def test_02_list_cpvm_vm(self): """Test List console proxy VMs """ @@ -211,111 +237,123 @@ class TestSSVMs(cloudstackTestCase): # 5. DNS entries must match those given for the zone list_cpvm_response = list_ssvms( - self.apiclient, - systemvmtype='consoleproxy', - state='Running', - ) + self.apiclient, + systemvmtype='consoleproxy', + state='Running', + ) self.assertEqual( - isinstance(list_cpvm_response, list), - True, - "Check list response returns a valid list" - ) - #Verify CPVM response + isinstance(list_cpvm_response, list), + True, + "Check list response returns a valid list" + ) + # Verify CPVM response self.assertNotEqual( - len(list_cpvm_response), - 0, - "Check list System VMs response" - ) + len(list_cpvm_response), + 0, + "Check list System VMs response" + ) list_zones_response = list_zones(self.apiclient) # Number of Console Proxy VMs = No of Zones - + self.assertEqual( - isinstance(list_zones_response, list), - True, - "Check list response returns a valid list" - ) - + isinstance(list_zones_response, list), + True, + "Check list response returns a valid list" + ) + self.debug("Number of zones: %s" % len(list_zones_response)) self.debug("Number of CPVMs: %s" % len(list_cpvm_response)) self.assertEqual( - len(list_cpvm_response), - len(list_zones_response), - "Check number of CPVMs with number of zones" - ) - #For each CPVM check private IP, public IP, link local IP and DNS + len(list_cpvm_response), + len(list_zones_response), + "Check number of CPVMs with number of zones" + ) + # For each CPVM check private IP, public IP, link local IP and DNS for cpvm in list_cpvm_response: self.debug("CPVM state: %s" % cpvm.state) self.assertEqual( - cpvm.state, - 'Running', - "Check whether state of CPVM is running" - ) + cpvm.state, + 'Running', + "Check whether state of CPVM is running" + ) self.assertEqual( - hasattr(cpvm, 'privateip'), - True, - "Check whether CPVM has private IP field" - ) + hasattr(cpvm, 'privateip'), + True, + "Check whether CPVM has private IP field" + ) self.assertEqual( - hasattr(cpvm, 'linklocalip'), - True, - "Check whether CPVM has link local IP field" - ) + hasattr(cpvm, 'linklocalip'), + True, + "Check whether CPVM has link local IP field" + ) self.assertEqual( - hasattr(cpvm, 'publicip'), - True, - "Check whether CPVM has public IP field" - ) - #Fetch corresponding ip ranges information from listVlanIpRanges + hasattr(cpvm, 'publicip'), + True, + "Check whether CPVM has public IP field" + ) + # Fetch corresponding ip ranges information from listVlanIpRanges ipranges_response = list_vlan_ipranges( - self.apiclient, - zoneid=cpvm.zoneid - ) + self.apiclient, + zoneid=cpvm.zoneid + ) self.assertEqual( - isinstance(ipranges_response, list), - True, - "Check list response returns a valid list" - ) + isinstance(ipranges_response, list), + True, + "Check list response returns a valid list" + ) iprange = ipranges_response[0] - #Fetch corresponding Physical Network of SSVM's Zone + # Fetch corresponding Physical Network of SSVM's Zone listphyntwk = PhysicalNetwork.list( - self.apiclient, - zoneid=cpvm.zoneid - ) - + self.apiclient, + zoneid=cpvm.zoneid + ) + # Execute the following assertion in all zones except EIP-ELB Zones - if not (self.zone.networktype.lower() == 'basic' and isinstance(NetScaler.list(self.apiclient,physicalnetworkid=listphyntwk[0].id), list) is True): + if not ( + self.zone.networktype.lower() == 'basic' and isinstance( + NetScaler.list( + self.apiclient, + physicalnetworkid=listphyntwk[0].id), + list) is True): self.assertEqual( - cpvm.gateway, - iprange.gateway, - "Check gateway with that of corresponding ip range" - ) + cpvm.gateway, + iprange.gateway, + "Check gateway with that of corresponding ip range" + ) - #Fetch corresponding zone information from listZones + # Fetch corresponding zone information from listZones zone_response = list_zones( - self.apiclient, - id=cpvm.zoneid - ) + self.apiclient, + id=cpvm.zoneid + ) self.assertEqual( - cpvm.dns1, - zone_response[0].dns1, - "Check DNS1 with that of corresponding zone" - ) + cpvm.dns1, + zone_response[0].dns1, + "Check DNS1 with that of corresponding zone" + ) self.assertEqual( - cpvm.dns2, - zone_response[0].dns2, - "Check DNS2 with that of corresponding zone" - ) + cpvm.dns2, + zone_response[0].dns2, + "Check DNS2 with that of corresponding zone" + ) return - @attr(tags = ["advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="true") + @attr( + tags=[ + "advanced", + "advancedns", + "smoke", + "basic", + "sg"], + required_hardware="true") def test_03_ssvm_internals(self): """Test SSVM Internals""" @@ -329,106 +367,119 @@ class TestSSVMs(cloudstackTestCase): # then the test is a failure list_ssvm_response = list_ssvms( - self.apiclient, - systemvmtype='secondarystoragevm', - state='Running', - zoneid=self.zone.id - ) + self.apiclient, + systemvmtype='secondarystoragevm', + state='Running', + zoneid=self.zone.id + ) self.assertEqual( - isinstance(list_ssvm_response, list), - True, - "Check list response returns a valid list" - ) + isinstance(list_ssvm_response, list), + True, + "Check list response returns a valid list" + ) ssvm = list_ssvm_response[0] hosts = list_hosts( - self.apiclient, - id=ssvm.hostid - ) + self.apiclient, + id=ssvm.hostid + ) self.assertEqual( - isinstance(hosts, list), - True, - "Check list response returns a valid list" - ) + isinstance(hosts, list), + True, + "Check list response returns a valid list" + ) host = hosts[0] self.debug("Running SSVM check script") if self.hypervisor.lower() in ('vmware', 'hyperv'): - #SSH into SSVMs is done via management server for Vmware and Hyper-V + # SSH into SSVMs is done via management server for Vmware and + # Hyper-V result = get_process_status( - self.apiclient.connection.mgtSvr, - 22, - self.apiclient.connection.user, - self.apiclient.connection.passwd, - ssvm.privateip, - "/usr/local/cloud/systemvm/ssvm-check.sh |grep -e ERROR -e WARNING -e FAIL", - hypervisor=self.hypervisor - ) + self.apiclient.connection.mgtSvr, + 22, + self.apiclient.connection.user, + self.apiclient.connection.passwd, + ssvm.privateip, + "/usr/local/cloud/systemvm/ssvm-check.sh |grep -e ERROR -e WARNING -e FAIL", + hypervisor=self.hypervisor) else: try: - host.user, host.passwd = get_host_credentials(self.config, host.ipaddress) + host.user, host.passwd = get_host_credentials( + self.config, host.ipaddress) result = get_process_status( - host.ipaddress, - 22, - host.user, - host.passwd, - ssvm.linklocalip, - "/usr/local/cloud/systemvm/ssvm-check.sh |grep -e ERROR -e WARNING -e FAIL" - ) + host.ipaddress, + 22, + host.user, + host.passwd, + ssvm.linklocalip, + "/usr/local/cloud/systemvm/ssvm-check.sh |grep -e ERROR -e WARNING -e FAIL") except KeyError: - self.skipTest("Marvin configuration has no host credentials to check router services") + self.skipTest( + "Marvin configuration has no host\ + credentials to check router services") res = str(result) self.debug("SSVM script output: %s" % res) self.assertEqual( - res.count("ERROR"), - 1, - "Check for Errors in tests" - ) + res.count("ERROR"), + 1, + "Check for Errors in tests" + ) self.assertEqual( - res.count("WARNING"), - 1, - "Check for warnings in tests" - ) + res.count("WARNING"), + 1, + "Check for warnings in tests" + ) - #Check status of cloud service + # Check status of cloud service if self.hypervisor.lower() in ('vmware', 'hyperv'): - #SSH into SSVMs is done via management server for Vmware and Hyper-V + # SSH into SSVMs is done via management server for Vmware and + # Hyper-V result = get_process_status( - self.apiclient.connection.mgtSvr, - 22, - self.apiclient.connection.user, - self.apiclient.connection.passwd, - ssvm.privateip, - "service cloud status", - hypervisor=self.hypervisor - ) + self.apiclient.connection.mgtSvr, + 22, + self.apiclient.connection.user, + self.apiclient.connection.passwd, + ssvm.privateip, + "service cloud status", + hypervisor=self.hypervisor + ) else: try: - host.user, host.passwd = get_host_credentials(self.config, host.ipaddress) + host.user, host.passwd = get_host_credentials( + self.config, host.ipaddress) result = get_process_status( - host.ipaddress, - 22, - host.user, - host.passwd, - ssvm.linklocalip, - "service cloud status" - ) + host.ipaddress, + 22, + host.user, + host.passwd, + ssvm.linklocalip, + "service cloud status" + ) except KeyError: - self.skipTest("Marvin configuration has no host credentials to check router services") + self.skipTest( + "Marvin configuration has no host\ + credentials to check router services") res = str(result) self.debug("Cloud Process status: %s" % res) # cloud.com service (type=secstorage) is running: process id: 2346 self.assertEqual( - res.count("is running"), - 1, - "Check cloud service is running or not" - ) + res.count("is running"), + 1, + "Check cloud service is running or not" + ) return - @attr(tags = ["advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="true") + @attr( + tags=[ + "advanced", + "advancedns", + "smoke", + "basic", + "sg"], + required_hardware="true") def test_04_cpvm_internals(self): """Test CPVM Internals""" @@ -440,78 +491,89 @@ class TestSSVMs(cloudstackTestCase): # running list_cpvm_response = list_ssvms( - self.apiclient, - systemvmtype='consoleproxy', - state='Running', - zoneid=self.zone.id - ) + self.apiclient, + systemvmtype='consoleproxy', + state='Running', + zoneid=self.zone.id + ) self.assertEqual( - isinstance(list_cpvm_response, list), - True, - "Check list response returns a valid list" - ) + isinstance(list_cpvm_response, list), + True, + "Check list response returns a valid list" + ) cpvm = list_cpvm_response[0] hosts = list_hosts( - self.apiclient, - id=cpvm.hostid - ) + self.apiclient, + id=cpvm.hostid + ) self.assertEqual( - isinstance(hosts, list), - True, - "Check list response returns a valid list" - ) + isinstance(hosts, list), + True, + "Check list response returns a valid list" + ) host = hosts[0] try: - telnet = telnetlib.Telnet( - str(self.apiclient.connection.mgtSvr), - '8250' - ) - self.debug("Telnet management server (IP: %s)" % - self.apiclient.connection.mgtSvr) + telnetlib.Telnet( + str(self.apiclient.connection.mgtSvr), + '8250' + ) + self.debug("Telnet management server (IP: %s)" % + self.apiclient.connection.mgtSvr) except Exception as e: self.fail( - "Telnet Access failed for %s: %s" % \ - (self.apiclient.connection.mgtSvr, e) - ) + "Telnet Access failed for %s: %s" % + (self.apiclient.connection.mgtSvr, e) + ) self.debug("Checking cloud process status") if self.hypervisor.lower() in ('vmware', 'hyperv'): - #SSH into SSVMs is done via management server for Vmware and Hyper-V + # SSH into SSVMs is done via management server for Vmware and + # Hyper-V result = get_process_status( - self.apiclient.connection.mgtSvr, - 22, - self.apiclient.connection.user, - self.apiclient.connection.passwd, - cpvm.privateip, - "service cloud status", - hypervisor=self.hypervisor - ) + self.apiclient.connection.mgtSvr, + 22, + self.apiclient.connection.user, + self.apiclient.connection.passwd, + cpvm.privateip, + "service cloud status", + hypervisor=self.hypervisor + ) else: try: - host.user, host.passwd = get_host_credentials(self.config, host.ipaddress) + host.user, host.passwd = get_host_credentials( + self.config, host.ipaddress) result = get_process_status( - host.ipaddress, - 22, - host.user, - host.passwd, - cpvm.linklocalip, - "service cloud status" - ) + host.ipaddress, + 22, + host.user, + host.passwd, + cpvm.linklocalip, + "service cloud status" + ) except KeyError: - self.skipTest("Marvin configuration has no host credentials to check router services") + self.skipTest( + "Marvin configuration has no host\ + credentials to check router services") res = str(result) self.debug("Cloud Process status: %s" % res) self.assertEqual( - res.count("is running"), - 1, - "Check cloud service is running or not" - ) + res.count("is running"), + 1, + "Check cloud service is running or not" + ) return - @attr(tags = ["advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="true") + @attr( + tags=[ + "advanced", + "advancedns", + "smoke", + "basic", + "sg"], + required_hardware="true") def test_05_stop_ssvm(self): """Test stop SSVM """ @@ -524,70 +586,81 @@ class TestSSVMs(cloudstackTestCase): # 3. If either of the two above steps fail the test is a failure list_ssvm_response = list_ssvms( - self.apiclient, - systemvmtype='secondarystoragevm', - state='Running', - zoneid=self.zone.id - ) + self.apiclient, + systemvmtype='secondarystoragevm', + state='Running', + zoneid=self.zone.id + ) self.assertEqual( - isinstance(list_ssvm_response, list), - True, - "Check list response returns a valid list" - ) + isinstance(list_ssvm_response, list), + True, + "Check list response returns a valid list" + ) ssvm = list_ssvm_response[0] hosts = list_hosts( - self.apiclient, - id=ssvm.hostid - ) + self.apiclient, + id=ssvm.hostid + ) self.assertEqual( - isinstance(hosts, list), - True, - "Check list response returns a valid list" - ) - host = hosts[0] + isinstance(hosts, list), + True, + "Check list response returns a valid list" + ) self.debug("Stopping SSVM: %s" % ssvm.id) cmd = stopSystemVm.stopSystemVmCmd() cmd.id = ssvm.id self.apiclient.stopSystemVm(cmd) - + timeout = self.services["timeout"] while True: list_ssvm_response = list_ssvms( - self.apiclient, - id=ssvm.id - ) + self.apiclient, + id=ssvm.id + ) if isinstance(list_ssvm_response, list): if list_ssvm_response[0].state == 'Running': break if timeout == 0: raise Exception("List SSVM call failed!") - + time.sleep(self.services["sleep"]) timeout = timeout - 1 - + self.assertEqual( - isinstance(list_ssvm_response, list), - True, - "Check list response returns a valid list" - ) + isinstance(list_ssvm_response, list), + True, + "Check list response returns a valid list" + ) ssvm_response = list_ssvm_response[0] self.debug("SSVM state after debug: %s" % ssvm_response.state) self.assertEqual( - ssvm_response.state, - 'Running', - "Check whether SSVM is running or not" - ) + ssvm_response.state, + 'Running', + "Check whether SSVM is running or not" + ) # Wait for the agent to be up self.waitForSystemVMAgent(ssvm_response.name) # Call above tests to ensure SSVM is properly running self.test_01_list_sec_storage_vm() + + # Wait for some time before running diagnostic scripts on SSVM + # as it may take some time to start all service properly + time.sleep(int(self.services["configurableData"]["systemVmDelay"])) + self.test_03_ssvm_internals() return - @attr(tags = ["advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="true") + @attr( + tags=[ + "advanced", + "advancedns", + "smoke", + "basic", + "sg"], + required_hardware="true") def test_06_stop_cpvm(self): """Test stop CPVM """ @@ -600,28 +673,27 @@ class TestSSVMs(cloudstackTestCase): # 3. If either of the two above steps fail the test is a failure list_cpvm_response = list_ssvms( - self.apiclient, - systemvmtype='consoleproxy', - state='Running', - zoneid=self.zone.id - ) + self.apiclient, + systemvmtype='consoleproxy', + state='Running', + zoneid=self.zone.id + ) self.assertEqual( - isinstance(list_cpvm_response, list), - True, - "Check list response returns a valid list" - ) + isinstance(list_cpvm_response, list), + True, + "Check list response returns a valid list" + ) cpvm = list_cpvm_response[0] hosts = list_hosts( - self.apiclient, - id=cpvm.hostid - ) + self.apiclient, + id=cpvm.hostid + ) self.assertEqual( - isinstance(hosts, list), - True, - "Check list response returns a valid list" - ) - host = hosts[0] + isinstance(hosts, list), + True, + "Check list response returns a valid list" + ) self.debug("Stopping CPVM: %s" % cpvm.id) cmd = stopSystemVm.stopSystemVmCmd() @@ -631,15 +703,15 @@ class TestSSVMs(cloudstackTestCase): timeout = self.services["timeout"] while True: list_cpvm_response = list_ssvms( - self.apiclient, - id=cpvm.id - ) + self.apiclient, + id=cpvm.id + ) if isinstance(list_cpvm_response, list): if list_cpvm_response[0].state == 'Running': break if timeout == 0: raise Exception("List CPVM call failed!") - + time.sleep(self.services["sleep"]) timeout = timeout - 1 @@ -648,20 +720,32 @@ class TestSSVMs(cloudstackTestCase): self.debug("CPVM state after debug: %s" % cpvm_response.state) self.assertEqual( - cpvm_response.state, - 'Running', - "Check whether CPVM is running or not" - ) + cpvm_response.state, + 'Running', + "Check whether CPVM is running or not" + ) # Wait for the agent to be up self.waitForSystemVMAgent(cpvm_response.name) # Call above tests to ensure CPVM is properly running self.test_02_list_cpvm_vm() + + # Wait for some time before running diagnostic scripts on SSVM + # as it may take some time to start all service properly + time.sleep(int(self.services["configurableData"]["systemVmDelay"])) + self.test_04_cpvm_internals() return - @attr(tags = ["advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="true") + @attr( + tags=[ + "advanced", + "advancedns", + "smoke", + "basic", + "sg"], + required_hardware="true") def test_07_reboot_ssvm(self): """Test reboot SSVM """ @@ -672,32 +756,31 @@ class TestSSVMs(cloudstackTestCase): # 3. The cloud process should still be running within the SSVM list_ssvm_response = list_ssvms( - self.apiclient, - systemvmtype='secondarystoragevm', - state='Running', - zoneid=self.zone.id - ) - + self.apiclient, + systemvmtype='secondarystoragevm', + state='Running', + zoneid=self.zone.id + ) + self.assertEqual( - isinstance(list_ssvm_response, list), - True, - "Check list response returns a valid list" - ) - + isinstance(list_ssvm_response, list), + True, + "Check list response returns a valid list" + ) + ssvm_response = list_ssvm_response[0] hosts = list_hosts( - self.apiclient, - id=ssvm_response.hostid - ) + self.apiclient, + id=ssvm_response.hostid + ) self.assertEqual( - isinstance(hosts, list), - True, - "Check list response returns a valid list" - ) - host = hosts[0] + isinstance(hosts, list), + True, + "Check list response returns a valid list" + ) - #Store the public & private IP values before reboot + # Store the public & private IP values before reboot old_public_ip = ssvm_response.publicip old_private_ip = ssvm_response.privateip @@ -709,46 +792,57 @@ class TestSSVMs(cloudstackTestCase): timeout = self.services["timeout"] while True: list_ssvm_response = list_ssvms( - self.apiclient, - id=ssvm_response.id - ) + self.apiclient, + id=ssvm_response.id + ) if isinstance(list_ssvm_response, list): if list_ssvm_response[0].state == 'Running': break if timeout == 0: raise Exception("List SSVM call failed!") - + time.sleep(self.services["sleep"]) timeout = timeout - 1 ssvm_response = list_ssvm_response[0] self.debug("SSVM State: %s" % ssvm_response.state) self.assertEqual( - 'Running', - str(ssvm_response.state), - "Check whether CPVM is running or not" - ) + 'Running', + str(ssvm_response.state), + "Check whether CPVM is running or not" + ) self.assertEqual( - ssvm_response.publicip, - old_public_ip, - "Check Public IP after reboot with that of before reboot" - ) + ssvm_response.publicip, + old_public_ip, + "Check Public IP after reboot with that of before reboot" + ) self.assertEqual( - ssvm_response.privateip, - old_private_ip, - "Check Private IP after reboot with that of before reboot" - ) + ssvm_response.privateip, + old_private_ip, + "Check Private IP after reboot with that of before reboot" + ) # Wait for the agent to be up self.waitForSystemVMAgent(ssvm_response.name) - #Call to verify cloud process is running + # Wait for some time before running diagnostic scripts on SSVM + # as it may take some time to start all service properly + time.sleep(int(self.services["configurableData"]["systemVmDelay"])) + + # Call to verify cloud process is running self.test_03_ssvm_internals() return - @attr(tags = ["advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="true") + @attr( + tags=[ + "advanced", + "advancedns", + "smoke", + "basic", + "sg"], + required_hardware="true") def test_08_reboot_cpvm(self): """Test reboot CPVM """ @@ -758,32 +852,30 @@ class TestSSVMs(cloudstackTestCase): # the same before and after reboot # 3. the cloud process should still be running within the CPVM - list_cpvm_response = list_ssvms( - self.apiclient, - systemvmtype='consoleproxy', - state='Running', - zoneid=self.zone.id - ) + self.apiclient, + systemvmtype='consoleproxy', + state='Running', + zoneid=self.zone.id + ) self.assertEqual( - isinstance(list_cpvm_response, list), - True, - "Check list response returns a valid list" - ) + isinstance(list_cpvm_response, list), + True, + "Check list response returns a valid list" + ) cpvm_response = list_cpvm_response[0] hosts = list_hosts( - self.apiclient, - id=cpvm_response.hostid - ) + self.apiclient, + id=cpvm_response.hostid + ) self.assertEqual( - isinstance(hosts, list), - True, - "Check list response returns a valid list" - ) - host = hosts[0] + isinstance(hosts, list), + True, + "Check list response returns a valid list" + ) - #Store the public & private IP values before reboot + # Store the public & private IP values before reboot old_public_ip = cpvm_response.publicip old_private_ip = cpvm_response.privateip @@ -796,15 +888,15 @@ class TestSSVMs(cloudstackTestCase): timeout = self.services["timeout"] while True: list_cpvm_response = list_ssvms( - self.apiclient, - id=cpvm_response.id - ) + self.apiclient, + id=cpvm_response.id + ) if isinstance(list_cpvm_response, list): if list_cpvm_response[0].state == 'Running': break if timeout == 0: raise Exception("List CPVM call failed!") - + time.sleep(self.services["sleep"]) timeout = timeout - 1 @@ -812,30 +904,41 @@ class TestSSVMs(cloudstackTestCase): self.debug("CPVM state: %s" % cpvm_response.state) self.assertEqual( - 'Running', - str(cpvm_response.state), - "Check whether CPVM is running or not" - ) + 'Running', + str(cpvm_response.state), + "Check whether CPVM is running or not" + ) self.assertEqual( - cpvm_response.publicip, - old_public_ip, - "Check Public IP after reboot with that of before reboot" - ) + cpvm_response.publicip, + old_public_ip, + "Check Public IP after reboot with that of before reboot" + ) self.assertEqual( - cpvm_response.privateip, - old_private_ip, - "Check Private IP after reboot with that of before reboot" - ) + cpvm_response.privateip, + old_private_ip, + "Check Private IP after reboot with that of before reboot" + ) # Wait for the agent to be up self.waitForSystemVMAgent(cpvm_response.name) - #Call to verify cloud process is running + # Wait for some time before running diagnostic scripts on SSVM + # as it may take some time to start all service properly + time.sleep(int(self.services["configurableData"]["systemVmDelay"])) + + # Call to verify cloud process is running self.test_04_cpvm_internals() return - @attr(tags = ["advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="true") + @attr( + tags=[ + "advanced", + "advancedns", + "smoke", + "basic", + "sg"], + required_hardware="true") def test_09_destroy_ssvm(self): """Test destroy SSVM """ @@ -848,16 +951,16 @@ class TestSSVMs(cloudstackTestCase): # 4. cloud process within SSVM must be up and running list_ssvm_response = list_ssvms( - self.apiclient, - systemvmtype='secondarystoragevm', - state='Running', - zoneid=self.zone.id - ) + self.apiclient, + systemvmtype='secondarystoragevm', + state='Running', + zoneid=self.zone.id + ) self.assertEqual( - isinstance(list_ssvm_response, list), - True, - "Check list response returns a valid list" - ) + isinstance(list_ssvm_response, list), + True, + "Check list response returns a valid list" + ) ssvm_response = list_ssvm_response[0] old_name = ssvm_response.name @@ -870,16 +973,16 @@ class TestSSVMs(cloudstackTestCase): timeout = self.services["timeout"] while True: list_ssvm_response = list_ssvms( - self.apiclient, - zoneid=self.zone.id, - systemvmtype='secondarystoragevm' - ) + self.apiclient, + zoneid=self.zone.id, + systemvmtype='secondarystoragevm' + ) if isinstance(list_ssvm_response, list): if list_ssvm_response[0].state == 'Running': break if timeout == 0: raise Exception("List SSVM call failed!") - + time.sleep(self.services["sleep"]) timeout = timeout - 1 @@ -888,36 +991,47 @@ class TestSSVMs(cloudstackTestCase): # Verify Name, Public IP, Private IP and Link local IP # for newly created SSVM self.assertNotEqual( - ssvm_response.name, - old_name, - "Check SSVM new name with name of destroyed SSVM" - ) + ssvm_response.name, + old_name, + "Check SSVM new name with name of destroyed SSVM" + ) self.assertEqual( - hasattr(ssvm_response, 'privateip'), - True, - "Check whether SSVM has private IP field" - ) + hasattr(ssvm_response, 'privateip'), + True, + "Check whether SSVM has private IP field" + ) self.assertEqual( - hasattr(ssvm_response, 'linklocalip'), - True, - "Check whether SSVM has link local IP field" - ) + hasattr(ssvm_response, 'linklocalip'), + True, + "Check whether SSVM has link local IP field" + ) self.assertEqual( - hasattr(ssvm_response, 'publicip'), - True, - "Check whether SSVM has public IP field" - ) - + hasattr(ssvm_response, 'publicip'), + True, + "Check whether SSVM has public IP field" + ) + # Wait for the agent to be up self.waitForSystemVMAgent(ssvm_response.name) - #Call to verify cloud process is running + # Wait for some time before running diagnostic scripts on SSVM + # as it may take some time to start all service properly + time.sleep(int(self.services["configurableData"]["systemVmDelay"])) + + # Call to verify cloud process is running self.test_03_ssvm_internals() return - @attr(tags = ["advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="true") + @attr( + tags=[ + "advanced", + "advancedns", + "smoke", + "basic", + "sg"], + required_hardware="true") def test_10_destroy_cpvm(self): """Test destroy CPVM """ @@ -930,15 +1044,15 @@ class TestSSVMs(cloudstackTestCase): # 4. cloud process within CPVM must be up and running list_cpvm_response = list_ssvms( - self.apiclient, - systemvmtype='consoleproxy', - zoneid=self.zone.id - ) + self.apiclient, + systemvmtype='consoleproxy', + zoneid=self.zone.id + ) self.assertEqual( - isinstance(list_cpvm_response, list), - True, - "Check list response returns a valid list" - ) + isinstance(list_cpvm_response, list), + True, + "Check list response returns a valid list" + ) cpvm_response = list_cpvm_response[0] old_name = cpvm_response.name @@ -951,16 +1065,16 @@ class TestSSVMs(cloudstackTestCase): timeout = self.services["timeout"] while True: list_cpvm_response = list_ssvms( - self.apiclient, - systemvmtype='consoleproxy', - zoneid=self.zone.id - ) + self.apiclient, + systemvmtype='consoleproxy', + zoneid=self.zone.id + ) if isinstance(list_cpvm_response, list): if list_cpvm_response[0].state == 'Running': break if timeout == 0: raise Exception("List CPVM call failed!") - + time.sleep(self.services["sleep"]) timeout = timeout - 1 @@ -969,31 +1083,35 @@ class TestSSVMs(cloudstackTestCase): # Verify Name, Public IP, Private IP and Link local IP # for newly created CPVM self.assertNotEqual( - cpvm_response.name, - old_name, - "Check SSVM new name with name of destroyed CPVM" - ) + cpvm_response.name, + old_name, + "Check SSVM new name with name of destroyed CPVM" + ) self.assertEqual( - hasattr(cpvm_response, 'privateip'), - True, - "Check whether CPVM has private IP field" - ) + hasattr(cpvm_response, 'privateip'), + True, + "Check whether CPVM has private IP field" + ) self.assertEqual( - hasattr(cpvm_response, 'linklocalip'), - True, - "Check whether CPVM has link local IP field" - ) + hasattr(cpvm_response, 'linklocalip'), + True, + "Check whether CPVM has link local IP field" + ) self.assertEqual( - hasattr(cpvm_response, 'publicip'), - True, - "Check whether CPVM has public IP field" - ) - + hasattr(cpvm_response, 'publicip'), + True, + "Check whether CPVM has public IP field" + ) + # Wait for the agent to be up self.waitForSystemVMAgent(cpvm_response.name) - #Call to verify cloud process is running + # Wait for some time before running diagnostic scripts on SSVM + # as it may take some time to start all service properly + time.sleep(int(self.services["configurableData"]["systemVmDelay"])) + + # Call to verify cloud process is running self.test_04_cpvm_internals() return diff --git a/tools/marvin/marvin/config/test_data.py b/tools/marvin/marvin/config/test_data.py index 4ab7e56e825..ba2fb24840e 100644 --- a/tools/marvin/marvin/config/test_data.py +++ b/tools/marvin/marvin/config/test_data.py @@ -1460,6 +1460,7 @@ test_data = { "publicport": 22, "username": "root", "password": "password", - } + }, + "systemVmDelay": 120 } } From f18e00abf148423797f911e3ec7764c21bac6a00 Mon Sep 17 00:00:00 2001 From: SrikanteswaraRao Talluri Date: Tue, 23 Dec 2014 14:29:47 +0530 Subject: [PATCH 019/391] Revert "CLOUDSTACK-7762 -[Automation] - Fix test failure for test_02_revert_vm_snapshots in smoke/test_vm_snapshots.py" This reverts commit f510ef995baaa9addefc22ff0330cd51dee1dd95. --- test/integration/smoke/test_vm_snapshots.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/integration/smoke/test_vm_snapshots.py b/test/integration/smoke/test_vm_snapshots.py index ef1353a6df9..131da998508 100644 --- a/test/integration/smoke/test_vm_snapshots.py +++ b/test/integration/smoke/test_vm_snapshots.py @@ -202,9 +202,6 @@ class TestVmSnapshot(cloudstackTestCase): "Check the snapshot of vm is ready!" ) - # Stop Virtual machine befor reverting VM to a snapshot taken without memory - self.virtual_machine.stop(self.apiclient) - VmSnapshot.revertToSnapshot(self.apiclient, list_snapshot_response[0].id) list_vm_response = list_virtual_machines( From 125c1c27fc0ebe3781fd2fb63e2d2dd5689c9fa0 Mon Sep 17 00:00:00 2001 From: Gaurav Aradhye Date: Fri, 19 Dec 2014 17:38:06 +0530 Subject: [PATCH 020/391] CLOUDSTACK-8098: Fixed VM snapshot issue in smoke/test_vm_snapshots.py Signed-off-by: SrikanteswaraRao Talluri --- test/integration/smoke/test_vm_snapshots.py | 100 ++++++++++---------- 1 file changed, 51 insertions(+), 49 deletions(-) diff --git a/test/integration/smoke/test_vm_snapshots.py b/test/integration/smoke/test_vm_snapshots.py index 131da998508..94c0f336e0b 100644 --- a/test/integration/smoke/test_vm_snapshots.py +++ b/test/integration/smoke/test_vm_snapshots.py @@ -19,7 +19,6 @@ from marvin.codes import FAILED, KVM from nose.plugins.attrib import attr from marvin.cloudstackTestCase import cloudstackTestCase, unittest -from marvin.cloudstackAPI import startVirtualMachine from marvin.lib.utils import random_gen, cleanup_resources from marvin.lib.base import (Account, ServiceOffering, @@ -27,10 +26,10 @@ from marvin.lib.base import (Account, VmSnapshot) from marvin.lib.common import (get_zone, get_domain, - get_template, - list_virtual_machines) + get_template) import time + class TestVmSnapshot(cloudstackTestCase): @classmethod @@ -39,7 +38,8 @@ class TestVmSnapshot(cloudstackTestCase): hypervisor = testClient.getHypervisorInfo() if hypervisor.lower() in (KVM.lower(), "hyperv", "lxc"): - raise unittest.SkipTest("VM snapshot feature is not supported on KVM, Hyper-V or LXC") + raise unittest.SkipTest( + "VM snapshot feature is not supported on KVM, Hyper-V or LXC") cls.apiclient = testClient.getApiClient() cls.services = testClient.getParsedTestDataConfig() @@ -48,12 +48,13 @@ class TestVmSnapshot(cloudstackTestCase): cls.zone = get_zone(cls.apiclient, testClient.getZoneForTests()) template = get_template( - cls.apiclient, - cls.zone.id, - cls.services["ostype"] - ) + cls.apiclient, + cls.zone.id, + cls.services["ostype"] + ) if template == FAILED: - assert False, "get_template() failed to return template with description %s" % cls.services["ostype"] + assert False, "get_template() failed to return template\ + with description %s" % cls.services["ostype"] cls.services["domainid"] = cls.domain.id cls.services["server"]["zoneid"] = cls.zone.id @@ -62,31 +63,31 @@ class TestVmSnapshot(cloudstackTestCase): # Create VMs, NAT Rules etc cls.account = Account.create( - cls.apiclient, - cls.services["account"], - domainid=cls.domain.id - ) + cls.apiclient, + cls.services["account"], + domainid=cls.domain.id + ) cls.service_offering = ServiceOffering.create( - cls.apiclient, - cls.services["service_offerings"] - ) + cls.apiclient, + cls.services["service_offerings"] + ) cls.virtual_machine = VirtualMachine.create( - cls.apiclient, - cls.services["server"], - templateid=template.id, - accountid=cls.account.name, - domainid=cls.account.domainid, - serviceofferingid=cls.service_offering.id, - mode=cls.zone.networktype - ) + cls.apiclient, + cls.services["server"], + templateid=template.id, + accountid=cls.account.name, + domainid=cls.account.domainid, + serviceofferingid=cls.service_offering.id, + mode=cls.zone.networktype + ) cls.random_data_0 = random_gen(size=100) cls.test_dir = "/tmp" cls.random_data = "random.data" cls._cleanup = [ - cls.service_offering, - cls.account, - ] + cls.service_offering, + cls.account, + ] return @classmethod @@ -122,9 +123,10 @@ class TestVmSnapshot(cloudstackTestCase): ssh_client = self.virtual_machine.get_ssh_client() cmds = [ - "echo %s > %s/%s" % (self.random_data_0, self.test_dir, self.random_data), - "cat %s/%s" % (self.test_dir, self.random_data) - ] + "echo %s > %s/%s" % + (self.random_data_0, self.test_dir, self.random_data), + "cat %s/%s" % + (self.test_dir, self.random_data)] for c in cmds: self.debug(c) @@ -183,7 +185,10 @@ class TestVmSnapshot(cloudstackTestCase): time.sleep(self.services["sleep"]) - list_snapshot_response = VmSnapshot.list(self.apiclient, vmid=self.virtual_machine.id, listall=True) + list_snapshot_response = VmSnapshot.list( + self.apiclient, + vmid=self.virtual_machine.id, + listall=True) self.assertEqual( isinstance(list_snapshot_response, list), @@ -202,24 +207,13 @@ class TestVmSnapshot(cloudstackTestCase): "Check the snapshot of vm is ready!" ) - VmSnapshot.revertToSnapshot(self.apiclient, list_snapshot_response[0].id) + self.virtual_machine.stop(self.apiclient) - list_vm_response = list_virtual_machines( + VmSnapshot.revertToSnapshot( self.apiclient, - id=self.virtual_machine.id - ) + list_snapshot_response[0].id) - self.assertEqual( - list_vm_response[0].state, - "Stopped", - "Check the state of vm is Stopped!" - ) - - cmd = startVirtualMachine.startVirtualMachineCmd() - cmd.id = list_vm_response[0].id - self.apiclient.startVirtualMachine(cmd) - - time.sleep(self.services["sleep"]) + self.virtual_machine.start(self.apiclient) try: ssh_client = self.virtual_machine.get_ssh_client(reconnect=True) @@ -248,7 +242,10 @@ class TestVmSnapshot(cloudstackTestCase): """Test to delete vm snapshots """ - list_snapshot_response = VmSnapshot.list(self.apiclient, vmid=self.virtual_machine.id, listall=True) + list_snapshot_response = VmSnapshot.list( + self.apiclient, + vmid=self.virtual_machine.id, + listall=True) self.assertEqual( isinstance(list_snapshot_response, list), @@ -260,11 +257,16 @@ class TestVmSnapshot(cloudstackTestCase): None, "Check if snapshot exists in ListSnapshot" ) - VmSnapshot.deleteVMSnapshot(self.apiclient, list_snapshot_response[0].id) + VmSnapshot.deleteVMSnapshot( + self.apiclient, + list_snapshot_response[0].id) time.sleep(self.services["sleep"] * 3) - list_snapshot_response = VmSnapshot.list(self.apiclient, vmid=self.virtual_machine.id, listall=True) + list_snapshot_response = VmSnapshot.list( + self.apiclient, + vmid=self.virtual_machine.id, + listall=True) self.assertEqual( list_snapshot_response, From ddcbfb43bc0cb76ad32c81c74416cf0c98d3eb3e Mon Sep 17 00:00:00 2001 From: Chandan Purushothama Date: Sat, 25 Oct 2014 07:33:10 -0700 Subject: [PATCH 021/391] CLOUDSTACK-7788: Fixed the script 'test_dynamic_compute_offering.py' to be run only on hardware --- .../component/test_dynamic_compute_offering.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/integration/component/test_dynamic_compute_offering.py b/test/integration/component/test_dynamic_compute_offering.py index d96fd84075f..1e475793fd5 100644 --- a/test/integration/component/test_dynamic_compute_offering.py +++ b/test/integration/component/test_dynamic_compute_offering.py @@ -757,7 +757,7 @@ class TestScaleVmDynamicServiceOffering(cloudstackTestCase): return @data(ADMIN_ACCOUNT, USER_ACCOUNT) - @attr(tags=["basic","advanced"]) + @attr(tags=["basic","advanced"],required_hardware="true") def test_change_so_running_vm_static_to_static(self, value): """Test scale running VM from static offering to static offering""" @@ -817,7 +817,7 @@ class TestScaleVmDynamicServiceOffering(cloudstackTestCase): return @data(ADMIN_ACCOUNT, USER_ACCOUNT) - @attr(tags=["basic","advanced"]) + @attr(tags=["basic","advanced"],required_hardware="true") def test_change_so_running_vm_static_to_dynamic(self, value): """Test scale running VM from static offering to dynamic offering""" @@ -900,7 +900,7 @@ class TestScaleVmDynamicServiceOffering(cloudstackTestCase): return @data(ADMIN_ACCOUNT, USER_ACCOUNT) - @attr(tags=["basic","advanced"]) + @attr(tags=["basic","advanced"],required_hardware="true") def test_change_so_running_vm_dynamic_to_static(self, value): """Test scale running VM from dynamic offering to static offering""" @@ -963,7 +963,7 @@ class TestScaleVmDynamicServiceOffering(cloudstackTestCase): return @data(ADMIN_ACCOUNT, USER_ACCOUNT) - @attr(tags=["basic","advanced"]) + @attr(tags=["basic","advanced"],required_hardware="true") def test_change_so_running_vm_dynamic_to_dynamic(self, value): """Test scale running VM from dynamic offering to dynamic offering""" @@ -1378,7 +1378,7 @@ class TestAffinityGroup(cloudstackTestCase): return @data(ADMIN_ACCOUNT, USER_ACCOUNT) - @attr(tags=["basic","advanced"], BugId="7180") + @attr(tags=["basic","advanced"], BugId="7180", required_hardware="true") def test_deploy_VM_with_affinity_group(self, value): """Test deploy VMs with affinity group and dynamic compute offering""" From 5a6eb3b6e186fc5e45216647e8f521ace7f11ec3 Mon Sep 17 00:00:00 2001 From: Ashutosh K Date: Mon, 22 Dec 2014 15:40:39 +0530 Subject: [PATCH 022/391] CLOUDSTACK-8099: Fixed missing import in test_dynamic_compute_offering.py Signed-off-by: SrikanteswaraRao Talluri --- .../test_dynamic_compute_offering.py | 944 ++++++++++++------ 1 file changed, 620 insertions(+), 324 deletions(-) diff --git a/test/integration/component/test_dynamic_compute_offering.py b/test/integration/component/test_dynamic_compute_offering.py index 1e475793fd5..5215f844c01 100644 --- a/test/integration/component/test_dynamic_compute_offering.py +++ b/test/integration/component/test_dynamic_compute_offering.py @@ -17,13 +17,15 @@ """ Tests for Dynamic Compute Offering Feature - Test Plan: https://cwiki.apache.org/confluence/display/CLOUDSTACK/Dynamic+ComputeOffering + Test Plan: https://cwiki.apache.org/confluence/display/CLOUDSTACK + /Dynamic+ComputeOffering Issue Link: https://issues.apache.org/jira/browse/CLOUDSTACK-6147 - Feature Specifications: https://cwiki.apache.org/confluence/display/CLOUDSTACK/Dynamic+Compute+Offering+FS + Feature Specifications: https://cwiki.apache.org/confluence/display/ + CLOUDSTACK/Dynamic+Compute+Offering+FS """ -from marvin.cloudstackTestCase import cloudstackTestCase +from marvin.cloudstackTestCase import cloudstackTestCase, unittest from marvin.lib.utils import (cleanup_resources, validateList, random_gen, @@ -43,8 +45,10 @@ from nose.plugins.attrib import attr from marvin.codes import PASS, ADMIN_ACCOUNT, USER_ACCOUNT, FAILED from ddt import ddt, data + @ddt class TestDynamicServiceOffering(cloudstackTestCase): + """Test Dynamic Service Offerings """ @@ -65,7 +69,8 @@ class TestDynamicServiceOffering(cloudstackTestCase): cls.services["ostype"] ) if cls.template == FAILED: - assert False, "get_template() failed to return template with description %s" % cls.services["ostype"] + assert False, "get_template() failed to return template\ + with description %s" % cls.services["ostype"] cls.services["virtual_machine"]["zoneid"] = cls.zone.id cls.services["virtual_machine"]["template"] = cls.template.id cls._cleanup = [] @@ -101,14 +106,14 @@ class TestDynamicServiceOffering(cloudstackTestCase): raise Exception("Warning: Exception during cleanup : %s" % e) return - @attr(tags=["basic","advanced"]) + @attr(tags=["basic", "advanced"]) def test_create_normal_compute_offering(self): """ Create normal compute offering with non zero values for cpu, cpu number and memory""" # Steps: - # 1. Create normal compute offering with non zero values for cpu number, - # cpu speed, memory + # 1. Create normal compute offering with non zero values + # for cpu number, cpu speed, memory # Validations: # 1. Compute offering should be created @@ -117,16 +122,20 @@ class TestDynamicServiceOffering(cloudstackTestCase): self.services["service_offering"]["cpuspeed"] = 256 self.services["service_offering"]["memory"] = 128 - serviceOffering = ServiceOffering.create(self.api_client, - self.services["service_offering"] - ) - self.assertEqual(verifyComputeOfferingCreation(self.apiclient, serviceOffering.id), - PASS, "Compute Offering verification failed") + serviceOffering = ServiceOffering.create( + self.api_client, + self.services["service_offering"]) + self.assertEqual( + verifyComputeOfferingCreation( + self.apiclient, + serviceOffering.id), + PASS, + "Compute Offering verification failed") self.cleanup_co.append(serviceOffering) return - @attr(tags=["basic","advanced"]) + @attr(tags=["basic", "advanced"]) def test_create_dynamic_compute_offering(self): """ Create dynamic compute offering with cpunumber, cpuspeed and memory not specified""" @@ -142,16 +151,20 @@ class TestDynamicServiceOffering(cloudstackTestCase): self.services["service_offering"]["cpuspeed"] = "" self.services["service_offering"]["memory"] = "" - serviceOffering = ServiceOffering.create(self.api_client, - self.services["service_offering"] - ) - self.assertEqual(verifyComputeOfferingCreation(self.apiclient, serviceOffering.id), - PASS, "Compute Offering verification failed") + serviceOffering = ServiceOffering.create( + self.api_client, + self.services["service_offering"]) + self.assertEqual( + verifyComputeOfferingCreation( + self.apiclient, + serviceOffering.id), + PASS, + "Compute Offering verification failed") self.cleanup_co.append(serviceOffering) return - @attr(tags=["basic","advanced"]) + @attr(tags=["basic", "advanced"]) def test_create_dynamic_compute_offering_no_cpunumber(self): """ Create dynamic compute offering with only cpunumber unspecified""" @@ -163,16 +176,17 @@ class TestDynamicServiceOffering(cloudstackTestCase): self.services["service_offering"]["memory"] = 128 try: - serviceOffering = ServiceOffering.create(self.api_client, - self.services["service_offering"] - ) + serviceOffering = ServiceOffering.create( + self.api_client, + self.services["service_offering"]) self.cleanup_co.append(serviceOffering) - self.fail("Compute Offering creation succeded, it should have failed") + self.fail( + "Compute Offering creation succeded, it should have failed") except Exception: self.debug("Compute Offering Creation failed as expected") return - @attr(tags=["basic","advanced"]) + @attr(tags=["basic", "advanced"]) def test_create_dynamic_compute_offering_no_cpuspeed(self): """ Create dynamic compute offering with only cpuspeed unspecified""" @@ -184,16 +198,17 @@ class TestDynamicServiceOffering(cloudstackTestCase): self.services["service_offering"]["memory"] = 128 try: - serviceOffering = ServiceOffering.create(self.api_client, - self.services["service_offering"] - ) + serviceOffering = ServiceOffering.create( + self.api_client, + self.services["service_offering"]) self.cleanup_co.append(serviceOffering) - self.fail("Compute Offering creation succeded, it should have failed") + self.fail( + "Compute Offering creation succeded, it should have failed") except Exception: self.debug("Compute Offering Creation failed as expected") return - @attr(tags=["basic","advanced"]) + @attr(tags=["basic", "advanced"]) def test_create_dynamic_compute_offering_no_memory(self): """ Create dynamic compute offering with only memory unspecified""" @@ -205,17 +220,18 @@ class TestDynamicServiceOffering(cloudstackTestCase): self.services["service_offering"]["memory"] = "" try: - serviceOffering = ServiceOffering.create(self.api_client, - self.services["service_offering"] - ) + serviceOffering = ServiceOffering.create( + self.api_client, + self.services["service_offering"]) self.cleanup_co.append(serviceOffering) - self.fail("Compute Offering creation succeded, it should have failed") + self.fail( + "Compute Offering creation succeded, it should have failed") except Exception: self.debug("Compute Offering Creation failed as expected") return @data(ADMIN_ACCOUNT, USER_ACCOUNT) - @attr(tags=["basic","advanced"]) + @attr(tags=["basic", "advanced"]) def test_deploy_virtual_machines_static_offering(self, value): """Test deploy VM with static offering""" @@ -223,22 +239,26 @@ class TestDynamicServiceOffering(cloudstackTestCase): # 1. Create admin/user account and create its user api client # 2. Create a static compute offering # 3. Deploy a VM with account api client and static service offering - # 4. Repeat step 3 but also pass custom values for cpu number, cpu speed and memory - # while deploying VM + # 4. Repeat step 3 but also pass custom values for cpu number, + # cpu speed and memory while deploying VM # Validations: # 1. Step 3 should succeed # 2. Step 4 should fail - isadmin=True + isadmin = True if value == USER_ACCOUNT: - isadmin=False + isadmin = False # Create Account - self.account = Account.create(self.apiclient,self.services["account"],domainid=self.domain.id, admin=isadmin) + self.account = Account.create( + self.apiclient, + self.services["account"], + domainid=self.domain.id, + admin=isadmin) apiclient = self.testClient.getUserApiClient( - UserName=self.account.name, - DomainName=self.account.domain) + UserName=self.account.name, + DomainName=self.account.domain) self.cleanup.append(self.account) # Create service offering @@ -246,34 +266,41 @@ class TestDynamicServiceOffering(cloudstackTestCase): self.services["service_offering"]["cpuspeed"] = 256 self.services["service_offering"]["memory"] = 128 - serviceOffering = ServiceOffering.create(self.apiclient, - self.services["service_offering"]) + serviceOffering = ServiceOffering.create( + self.apiclient, + self.services["service_offering"]) self.cleanup_co.append(serviceOffering) # Deploy VM with static service offering try: - VirtualMachine.create(apiclient,self.services["virtual_machine"], - serviceofferingid=serviceOffering.id, - accountid=self.account.name,domainid=self.account.domainid) + VirtualMachine.create( + apiclient, + self.services["virtual_machine"], + serviceofferingid=serviceOffering.id, + accountid=self.account.name, + domainid=self.account.domainid) except Exception as e: self.fail("vm creation failed: %s" % e) # Deploy VM with static service offering, also with custom values try: - VirtualMachine.create(apiclient,self.services["virtual_machine"], + VirtualMachine.create( + apiclient, + self.services["virtual_machine"], serviceofferingid=serviceOffering.id, customcpunumber=4, customcpuspeed=512, custommemory=256, - accountid=self.account.name,domainid=self.account.domainid) + accountid=self.account.name, + domainid=self.account.domainid) self.fail("VM creation should have failed, it succeeded") except Exception as e: self.debug("vm creation failed as expected: %s" % e) return @data(ADMIN_ACCOUNT, USER_ACCOUNT) - @attr(tags=["basic","advanced"]) + @attr(tags=["basic", "advanced"]) def test_deploy_virtual_machines_dynamic_offering(self, value): """Test deploy VM with dynamic compute offering""" @@ -281,26 +308,31 @@ class TestDynamicServiceOffering(cloudstackTestCase): # 1. Create admin/user account and create its user api client # 2. Create a dynamic service offering # 3. Deploy a VM with account api client and dynamic service offering - # without providing custom values for cpu number, cpu speed and memory - # 4. Deploy a VM with account api client and dynamic service offering providing - # custom values for cpu number, cpu speed and memory - # 5. Deploy a VM with account api client and dynamic service offering providing - # custom values only for cpu number + # without providing custom values for cpu number, cpu speed and + # memory + # 4. Deploy a VM with account api client and dynamic service offering + # providing custom values for cpu number, cpu speed and memory + # 5. Deploy a VM with account api client and dynamic service offering + # providing custom values only for cpu number # Validations: # 1. Step 3 should fail # 2. Step 4 should succeed # 3. Step 5 should fail - isadmin=True + isadmin = True if value == USER_ACCOUNT: - isadmin=False + isadmin = False # Create Account and its api client - self.account = Account.create(self.apiclient,self.services["account"],domainid=self.domain.id, admin=isadmin) + self.account = Account.create( + self.apiclient, + self.services["account"], + domainid=self.domain.id, + admin=isadmin) apiclient = self.testClient.getUserApiClient( - UserName=self.account.name, - DomainName=self.account.domain) + UserName=self.account.name, + DomainName=self.account.domain) self.cleanup.append(self.account) # Create dynamic service offering @@ -308,17 +340,22 @@ class TestDynamicServiceOffering(cloudstackTestCase): self.services["service_offering"]["cpuspeed"] = "" self.services["service_offering"]["memory"] = "" - serviceOffering = ServiceOffering.create(self.apiclient, - self.services["service_offering"]) + serviceOffering = ServiceOffering.create( + self.apiclient, + self.services["service_offering"]) self.cleanup_co.append(serviceOffering) - # Deploy VM with dynamic compute offering without providing custom values for + # Deploy VM with dynamic compute offering without providing + # custom values for # cpu number, cpu speed and memory try: - VirtualMachine.create(apiclient,self.services["virtual_machine"], - serviceofferingid=serviceOffering.id, - accountid=self.account.name,domainid=self.account.domainid) + VirtualMachine.create( + apiclient, + self.services["virtual_machine"], + serviceofferingid=serviceOffering.id, + accountid=self.account.name, + domainid=self.account.domainid) self.fail("VM creation succeded, it should have failed") except Exception as e: self.debug("vm creation failed as expected with error: %s" % e) @@ -326,29 +363,35 @@ class TestDynamicServiceOffering(cloudstackTestCase): # Deploy VM with dynamic compute offering providing custom values for # cpu number, cpu speed and memory try: - VirtualMachine.create(apiclient,self.services["virtual_machine"], + VirtualMachine.create( + apiclient, + self.services["virtual_machine"], serviceofferingid=serviceOffering.id, customcpunumber=2, customcpuspeed=256, custommemory=128, - accountid=self.account.name,domainid=self.account.domainid) + accountid=self.account.name, + domainid=self.account.domainid) except Exception as e: self.fail("vm creation failed: %s" % e) - # Deploy VM with dynamic compute offering providing custom values for only - # cpu number + # Deploy VM with dynamic compute offering providing custom values + # for only cpu number try: - VirtualMachine.create(apiclient,self.services["virtual_machine"], + VirtualMachine.create( + apiclient, + self.services["virtual_machine"], serviceofferingid=serviceOffering.id, customcpunumber=2, - accountid=self.account.name,domainid=self.account.domainid) + accountid=self.account.name, + domainid=self.account.domainid) self.fail("VM deployment should have failed, it succeded") except Exception as e: self.debug("vm creation failed as expected: %s" % e) return @data(ADMIN_ACCOUNT, USER_ACCOUNT) - @attr(tags=["basic","advanced"]) + @attr(tags=["basic", "advanced"]) def test_check_vm_stats(self, value): """Deploy VM with dynamic service offering and check VM stats""" @@ -359,15 +402,19 @@ class TestDynamicServiceOffering(cloudstackTestCase): # providing custom values for cpu number, cpu speed and memory # 4. List the VM and verify the dynamic parameters are same as passed - isadmin=True + isadmin = True if value == USER_ACCOUNT: - isadmin=False + isadmin = False # Create Account and api client - self.account = Account.create(self.apiclient,self.services["account"],domainid=self.domain.id, admin=isadmin) + self.account = Account.create( + self.apiclient, + self.services["account"], + domainid=self.domain.id, + admin=isadmin) apiclient = self.testClient.getUserApiClient( - UserName=self.account.name, - DomainName=self.account.domain) + UserName=self.account.name, + DomainName=self.account.domain) self.cleanup.append(self.account) # Create dynamic compute offering @@ -375,8 +422,9 @@ class TestDynamicServiceOffering(cloudstackTestCase): self.services["service_offering"]["cpuspeed"] = "" self.services["service_offering"]["memory"] = "" - serviceOffering = ServiceOffering.create(self.apiclient, - self.services["service_offering"]) + serviceOffering = ServiceOffering.create( + self.apiclient, + self.services["service_offering"]) self.cleanup_co.append(serviceOffering) @@ -387,41 +435,51 @@ class TestDynamicServiceOffering(cloudstackTestCase): # Deploy VM with dynamic service offering and the custom values try: - virtualMachine = VirtualMachine.create(apiclient,self.services["virtual_machine"], + virtualMachine = VirtualMachine.create( + apiclient, + self.services["virtual_machine"], serviceofferingid=serviceOffering.id, customcpunumber=customcpunumber, customcpuspeed=customcpuspeed, custommemory=custommemory, - accountid=self.account.name,domainid=self.account.domainid) + accountid=self.account.name, + domainid=self.account.domainid) except Exception as e: self.fail("vm creation failed: %s" % e) vmlist = VirtualMachine.list(self.apiclient, id=virtualMachine.id) - self.assertEqual(validateList(vmlist)[0], PASS, "vm list validation failed") + self.assertEqual( + validateList(vmlist)[0], + PASS, + "vm list validation failed") vm = vmlist[0] # Verify the custom values self.assertEqual(str(vm.cpunumber), str(customcpunumber), "vm cpu number %s\ - not matching with provided custom cpu number %s" % \ - (vm.cpunumber, customcpunumber)) + not matching with provided custom cpu number %s" % + (vm.cpunumber, customcpunumber)) self.assertEqual(str(vm.cpuspeed), str(customcpuspeed), "vm cpu speed %s\ - not matching with provided custom cpu speed %s" % \ - (vm.cpuspeed, customcpuspeed)) + not matching with provided custom cpu speed %s" % + (vm.cpuspeed, customcpuspeed)) self.assertEqual(str(vm.memory), str(custommemory), "vm memory %s\ - not matching with provided custom memory %s" % \ - (vm.memory, custommemory)) + not matching with provided custom memory %s" % + (vm.memory, custommemory)) return + @ddt class TestScaleVmDynamicServiceOffering(cloudstackTestCase): + """Test scaling VMs with dynamic Service Offerings """ @classmethod def setUpClass(cls): - cloudstackTestClient = super(TestScaleVmDynamicServiceOffering,cls).getClsTestClient() + cloudstackTestClient = super( + TestScaleVmDynamicServiceOffering, + cls).getClsTestClient() cls.api_client = cloudstackTestClient.getApiClient() cls.hypervisor = cloudstackTestClient.getHypervisorInfo() @@ -430,29 +488,35 @@ class TestScaleVmDynamicServiceOffering(cloudstackTestCase): # Get Zone, Domain and templates cls.domain = get_domain(cls.api_client) - cls.zone = get_zone(cls.api_client, cloudstackTestClient.getZoneForTests()) + cls.zone = get_zone( + cls.api_client, + cloudstackTestClient.getZoneForTests()) cls.mode = str(cls.zone.networktype).lower() cls.template = get_template( - cls.api_client, - cls.zone.id, - cls.services["ostype"] - ) + cls.api_client, + cls.zone.id, + cls.services["ostype"] + ) cls.services["virtual_machine"]["zoneid"] = cls.zone.id cls.services["virtual_machine"]["template"] = cls.template.id cls._cleanup = [] - cls.serviceOffering_static_1 = ServiceOffering.create(cls.api_client, - cls.services["service_offering"]) + cls.serviceOffering_static_1 = ServiceOffering.create( + cls.api_client, + cls.services["service_offering"]) cls._cleanup.append(cls.serviceOffering_static_1) if cls.hypervisor.lower() == "vmware": - virtual_machine = VirtualMachine.create(cls.api_client,cls.services["virtual_machine"], - serviceofferingid=cls.serviceOffering_static_1.id, mode=cls.zone.networktype) + virtual_machine = VirtualMachine.create( + cls.api_client, + cls.services["virtual_machine"], + serviceofferingid=cls.serviceOffering_static_1.id, + mode=cls.zone.networktype) cls._cleanup.append(virtual_machine) sshClient = virtual_machine.get_ssh_client() result = str( sshClient.execute("service vmware-tools status")).lower() - if not "running" in result: + if "running" not in result: cls.tearDownClass() raise unittest.SkipTest("Skipping scale VM operation because\ VMware tools are not installed on the VM") @@ -489,7 +553,7 @@ class TestScaleVmDynamicServiceOffering(cloudstackTestCase): return @data(ADMIN_ACCOUNT, USER_ACCOUNT) - @attr(tags=["basic","advanced"]) + @attr(tags=["basic", "advanced"]) def test_change_so_stopped_vm_static_to_static(self, value): """Test scale stopped VM from static offering to static offering""" @@ -503,51 +567,63 @@ class TestScaleVmDynamicServiceOffering(cloudstackTestCase): # Validations: # 1. Scaling operation should be successful - isadmin=True + isadmin = True if value == USER_ACCOUNT: - isadmin=False + isadmin = False try: # Create Account - self.account = Account.create(self.apiclient,self.services["account"],domainid=self.domain.id, admin=isadmin) + self.account = Account.create( + self.apiclient, + self.services["account"], + domainid=self.domain.id, + admin=isadmin) self.cleanup.append(self.account) apiclient = self.testClient.getUserApiClient( - UserName=self.account.name, - DomainName=self.account.domain) + UserName=self.account.name, + DomainName=self.account.domain) # Create static service offerings (Second offering should have - # one of the custom values greater than 1st one, scaling down is not allowed + # one of the custom values greater than 1st one, scaling down is + # not allowed self.services["service_offering"]["cpunumber"] = "2" self.services["service_offering"]["cpuspeed"] = "256" self.services["service_offering"]["memory"] = "128" - serviceOffering_static_1 = ServiceOffering.create(self.apiclient, - self.services["service_offering"]) + serviceOffering_static_1 = ServiceOffering.create( + self.apiclient, + self.services["service_offering"]) self.services["service_offering"]["cpunumber"] = "4" - serviceOffering_static_2 = ServiceOffering.create(self.apiclient, - self.services["service_offering"]) + serviceOffering_static_2 = ServiceOffering.create( + self.apiclient, + self.services["service_offering"]) self.cleanup_co.append(serviceOffering_static_1) self.cleanup_co.append(serviceOffering_static_2) # Deploy VM - virtualMachine = VirtualMachine.create(apiclient,self.services["virtual_machine"], + virtualMachine = VirtualMachine.create( + apiclient, + self.services["virtual_machine"], serviceofferingid=serviceOffering_static_1.id, - accountid=self.account.name,domainid=self.account.domainid) + accountid=self.account.name, + domainid=self.account.domainid) # Stop VM virtualMachine.stop(apiclient) # Scale VM to new static service offering - virtualMachine.scale(apiclient, serviceOfferingId=serviceOffering_static_2.id) + virtualMachine.scale( + apiclient, + serviceOfferingId=serviceOffering_static_2.id) except Exception as e: self.fail("Exception occured: %s" % e) return @data(ADMIN_ACCOUNT, USER_ACCOUNT) - @attr(tags=["basic","advanced"]) + @attr(tags=["basic", "advanced"]) def test_change_so_stopped_vm_static_to_dynamic(self, value): """Test scale stopped VM from static offering to dynamic offering""" @@ -559,22 +635,27 @@ class TestScaleVmDynamicServiceOffering(cloudstackTestCase): # 5. Scale VM with dynamic service offering providing all required # custom values # 6. Deploy another VM with static offring and stop the VM - # 7. Scale VM with dynamic service offering providing only custom cpu number + # 7. Scale VM with dynamic service offering providing only custom cpu + # number # Validations: # 1. Scale operation in step 5 should be successful # 2. Scale operation in step 7 should fail - isadmin=True + isadmin = True if value == USER_ACCOUNT: - isadmin=False + isadmin = False try: # Create Account and api client - self.account = Account.create(self.apiclient,self.services["account"],domainid=self.domain.id, admin=isadmin) + self.account = Account.create( + self.apiclient, + self.services["account"], + domainid=self.domain.id, + admin=isadmin) apiclient = self.testClient.getUserApiClient( - UserName=self.account.name, - DomainName=self.account.domain) + UserName=self.account.name, + DomainName=self.account.domain) self.cleanup.append(self.account) # Create static and dynamic service offerings @@ -582,50 +663,64 @@ class TestScaleVmDynamicServiceOffering(cloudstackTestCase): self.services["service_offering"]["cpuspeed"] = "256" self.services["service_offering"]["memory"] = "128" - serviceOffering_static = ServiceOffering.create(self.apiclient, - self.services["service_offering"]) - + serviceOffering_static = ServiceOffering.create( + self.apiclient, + self.services["service_offering"]) self.services["service_offering"]["cpunumber"] = "" self.services["service_offering"]["cpuspeed"] = "" self.services["service_offering"]["memory"] = "" - serviceOffering_dynamic = ServiceOffering.create(self.apiclient, - self.services["service_offering"]) + serviceOffering_dynamic = ServiceOffering.create( + self.apiclient, + self.services["service_offering"]) self.cleanup_co.append(serviceOffering_static) self.cleanup_co.append(serviceOffering_dynamic) # Deploy VM with static service offering - virtualMachine_1 = VirtualMachine.create(apiclient,self.services["virtual_machine"], + virtualMachine_1 = VirtualMachine.create( + apiclient, + self.services["virtual_machine"], serviceofferingid=serviceOffering_static.id, - accountid=self.account.name,domainid=self.account.domainid) + accountid=self.account.name, + domainid=self.account.domainid) # Stop VM virtualMachine_1.stop(apiclient) # Scale VM to dynamic service offering proving all custom values - virtualMachine_1.scale(apiclient, serviceOfferingId=serviceOffering_dynamic.id, - customcpunumber=4, customcpuspeed=256, custommemory=128) + virtualMachine_1.scale( + apiclient, + serviceOfferingId=serviceOffering_dynamic.id, + customcpunumber=4, + customcpuspeed=256, + custommemory=128) # Deploy VM with static service offering - virtualMachine_2 = VirtualMachine.create(apiclient,self.services["virtual_machine"], + virtualMachine_2 = VirtualMachine.create( + apiclient, + self.services["virtual_machine"], serviceofferingid=serviceOffering_static.id, - accountid=self.account.name,domainid=self.account.domainid) + accountid=self.account.name, + domainid=self.account.domainid) # Stop VM virtualMachine_2.stop(apiclient) except Exception as e: self.fail("Exception occuered: %s" % e) - # Scale VM to dynamic service offering proving only custom cpu number + # Scale VM to dynamic service offering proving only custom cpu + # number with self.assertRaises(Exception): - virtualMachine_2.scale(apiclient, serviceOfferingId=serviceOffering_dynamic.id, - customcpunumber=4) + virtualMachine_2.scale( + apiclient, + serviceOfferingId=serviceOffering_dynamic.id, + customcpunumber=4) return @data(ADMIN_ACCOUNT, USER_ACCOUNT) - @attr(tags=["basic","advanced"]) + @attr(tags=["basic", "advanced"]) def test_change_so_stopped_vm_dynamic_to_static(self, value): """Test scale stopped VM from dynamic offering to static offering""" @@ -639,16 +734,20 @@ class TestScaleVmDynamicServiceOffering(cloudstackTestCase): # Validations: # 1. Scale operation in step 5 should be successful - isadmin=True + isadmin = True if value == USER_ACCOUNT: - isadmin=False + isadmin = False try: # Create account and api client - self.account = Account.create(self.apiclient,self.services["account"],domainid=self.domain.id, admin=isadmin) + self.account = Account.create( + self.apiclient, + self.services["account"], + domainid=self.domain.id, + admin=isadmin) apiclient = self.testClient.getUserApiClient( - UserName=self.account.name, - DomainName=self.account.domain) + UserName=self.account.name, + DomainName=self.account.domain) self.cleanup.append(self.account) # Create dynamic and static service offering @@ -656,36 +755,45 @@ class TestScaleVmDynamicServiceOffering(cloudstackTestCase): self.services["service_offering"]["cpuspeed"] = "" self.services["service_offering"]["memory"] = "" - serviceOffering_dynamic = ServiceOffering.create(self.apiclient, - self.services["service_offering"]) + serviceOffering_dynamic = ServiceOffering.create( + self.apiclient, + self.services["service_offering"]) self.services["service_offering"]["cpunumber"] = "4" self.services["service_offering"]["cpuspeed"] = "256" self.services["service_offering"]["memory"] = "128" - serviceOffering_static = ServiceOffering.create(self.apiclient, - self.services["service_offering"]) + serviceOffering_static = ServiceOffering.create( + self.apiclient, + self.services["service_offering"]) self.cleanup_co.append(serviceOffering_static) self.cleanup_co.append(serviceOffering_dynamic) # Deploy VM with dynamic service offering - virtualMachine = VirtualMachine.create(apiclient,self.services["virtual_machine"], + virtualMachine = VirtualMachine.create( + apiclient, + self.services["virtual_machine"], serviceofferingid=serviceOffering_dynamic.id, - accountid=self.account.name,domainid=self.account.domainid, - customcpunumber=2, customcpuspeed=256, custommemory=128) + accountid=self.account.name, + domainid=self.account.domainid, + customcpunumber=2, + customcpuspeed=256, + custommemory=128) # Stop VM and verify that it is in stopped state virtualMachine.stop(apiclient) # Scale VM to static service offering - virtualMachine.scale(apiclient, serviceOfferingId=serviceOffering_static.id) + virtualMachine.scale( + apiclient, + serviceOfferingId=serviceOffering_static.id) except Exception as e: self.fail("Exception occured: %s" % e) return @data(ADMIN_ACCOUNT, USER_ACCOUNT) - @attr(tags=["basic","advanced"]) + @attr(tags=["basic", "advanced"]) def test_change_so_stopped_vm_dynamic_to_dynamic(self, value): """Test scale stopped VM from dynamic offering to dynamic offering""" @@ -704,16 +812,20 @@ class TestScaleVmDynamicServiceOffering(cloudstackTestCase): # 2. Scale operation in step 6 should be successful # 3. Scale operation in step 7 should fail - isadmin=True + isadmin = True if value == USER_ACCOUNT: - isadmin=False + isadmin = False try: # Create Account - self.account = Account.create(self.apiclient,self.services["account"],domainid=self.domain.id, admin=isadmin) + self.account = Account.create( + self.apiclient, + self.services["account"], + domainid=self.domain.id, + admin=isadmin) apiclient = self.testClient.getUserApiClient( - UserName=self.account.name, - DomainName=self.account.domain) + UserName=self.account.name, + DomainName=self.account.domain) self.cleanup.append(self.account) # Create dynamic service offerings @@ -721,43 +833,60 @@ class TestScaleVmDynamicServiceOffering(cloudstackTestCase): self.services["service_offering"]["cpuspeed"] = "" self.services["service_offering"]["memory"] = "" - serviceOffering_dynamic_1 = ServiceOffering.create(self.apiclient, - self.services["service_offering"]) + serviceOffering_dynamic_1 = ServiceOffering.create( + self.apiclient, + self.services["service_offering"]) - serviceOffering_dynamic_2 = ServiceOffering.create(self.apiclient, - self.services["service_offering"]) + serviceOffering_dynamic_2 = ServiceOffering.create( + self.apiclient, + self.services["service_offering"]) self.cleanup_co.append(serviceOffering_dynamic_1) self.cleanup_co.append(serviceOffering_dynamic_2) # Deploy VM with dynamic service offering - virtualMachine = VirtualMachine.create(apiclient,self.services["virtual_machine"], + virtualMachine = VirtualMachine.create( + apiclient, + self.services["virtual_machine"], serviceofferingid=serviceOffering_dynamic_1.id, - accountid=self.account.name,domainid=self.account.domainid, - customcpunumber=2, customcpuspeed=256, custommemory=128) + accountid=self.account.name, + domainid=self.account.domainid, + customcpunumber=2, + customcpuspeed=256, + custommemory=128) # Stop VM virtualMachine.stop(apiclient) # Scale VM with same dynamic service offering - virtualMachine.scale(apiclient, serviceOfferingId=serviceOffering_dynamic_1.id, - customcpunumber=4, customcpuspeed=512, custommemory=256) + virtualMachine.scale( + apiclient, + serviceOfferingId=serviceOffering_dynamic_1.id, + customcpunumber=4, + customcpuspeed=512, + custommemory=256) # Scale VM with other dynamic service offering - virtualMachine.scale(apiclient, serviceOfferingId=serviceOffering_dynamic_2.id, - customcpunumber=4, customcpuspeed=512, custommemory=256) + virtualMachine.scale( + apiclient, + serviceOfferingId=serviceOffering_dynamic_2.id, + customcpunumber=4, + customcpuspeed=512, + custommemory=256) except Exception as e: self.fail("Exception occured: %s" % e) # Scale VM with dynamic service offering proving custom value # only for cpu number with self.assertRaises(Exception): - virtualMachine.scale(apiclient, serviceOfferingId=serviceOffering_dynamic_1.id, - customcpunumber=4) + virtualMachine.scale( + apiclient, + serviceOfferingId=serviceOffering_dynamic_1.id, + customcpunumber=4) return @data(ADMIN_ACCOUNT, USER_ACCOUNT) - @attr(tags=["basic","advanced"],required_hardware="true") + @attr(tags=["basic", "advanced"],required_hardware="true") def test_change_so_running_vm_static_to_static(self, value): """Test scale running VM from static offering to static offering""" @@ -772,17 +901,22 @@ class TestScaleVmDynamicServiceOffering(cloudstackTestCase): hypervisor = get_hypervisor_type(self.apiclient) if hypervisor.lower() == "kvm": - self.skipTest("Scaling VM in running state is not supported on KVM") + self.skipTest( + "Scaling VM in running state is not supported on KVM") - isadmin=True + isadmin = True if value == USER_ACCOUNT: - isadmin=False + isadmin = False # Create Account and api client - self.account = Account.create(self.apiclient,self.services["account"],domainid=self.domain.id, admin=isadmin) + self.account = Account.create( + self.apiclient, + self.services["account"], + domainid=self.domain.id, + admin=isadmin) apiclient = self.testClient.getUserApiClient( - UserName=self.account.name, - DomainName=self.account.domain) + UserName=self.account.name, + DomainName=self.account.domain) self.cleanup.append(self.account) # Create static service offerings @@ -790,34 +924,41 @@ class TestScaleVmDynamicServiceOffering(cloudstackTestCase): self.services["service_offering"]["cpuspeed"] = "256" self.services["service_offering"]["memory"] = "128" - serviceOffering_static_1 = ServiceOffering.create(self.apiclient, - self.services["service_offering"]) + serviceOffering_static_1 = ServiceOffering.create( + self.apiclient, + self.services["service_offering"]) self.services["service_offering"]["cpunumber"] = "4" - serviceOffering_static_2 = ServiceOffering.create(self.apiclient, - self.services["service_offering"]) + serviceOffering_static_2 = ServiceOffering.create( + self.apiclient, + self.services["service_offering"]) self.cleanup_co.append(serviceOffering_static_1) self.cleanup_co.append(serviceOffering_static_2) # Deploy VM with static service offering try: - virtualMachine = VirtualMachine.create(apiclient,self.services["virtual_machine"], + virtualMachine = VirtualMachine.create( + apiclient, + self.services["virtual_machine"], serviceofferingid=serviceOffering_static_1.id, - accountid=self.account.name,domainid=self.account.domainid) + accountid=self.account.name, + domainid=self.account.domainid) except Exception as e: self.fail("vm creation failed: %s" % e) # Scale VM to other static service offering try: - virtualMachine.scale(apiclient, serviceOfferingId=serviceOffering_static_2.id) + virtualMachine.scale( + apiclient, + serviceOfferingId=serviceOffering_static_2.id) except Exception as e: self.fail("Failure while changing service offering: %s" % e) return @data(ADMIN_ACCOUNT, USER_ACCOUNT) - @attr(tags=["basic","advanced"],required_hardware="true") + @attr(tags=["basic", "advanced"],required_hardware="true") def test_change_so_running_vm_static_to_dynamic(self, value): """Test scale running VM from static offering to dynamic offering""" @@ -828,7 +969,8 @@ class TestScaleVmDynamicServiceOffering(cloudstackTestCase): # 4. Scale VM with dynamic service offering providing all required # custom values # 5. Deploy another VM with static offring - # 6. Scale VM with dynamic service offering providing only custom cpu number + # 6. Scale VM with dynamic service offering providing only custom cpu + # number # Validations: # 1. Scale operation in step 4 should be successful @@ -836,17 +978,22 @@ class TestScaleVmDynamicServiceOffering(cloudstackTestCase): hypervisor = get_hypervisor_type(self.apiclient) if hypervisor.lower() == "kvm": - self.skipTest("Scaling VM in running state is not supported on KVM") + self.skipTest( + "Scaling VM in running state is not supported on KVM") - isadmin=True + isadmin = True if value == USER_ACCOUNT: - isadmin=False + isadmin = False # Crate account and api client - self.account = Account.create(self.apiclient,self.services["account"],domainid=self.domain.id, admin=isadmin) + self.account = Account.create( + self.apiclient, + self.services["account"], + domainid=self.domain.id, + admin=isadmin) apiclient = self.testClient.getUserApiClient( - UserName=self.account.name, - DomainName=self.account.domain) + UserName=self.account.name, + DomainName=self.account.domain) self.cleanup.append(self.account) # Create static and dynamic service offerings @@ -854,53 +1001,70 @@ class TestScaleVmDynamicServiceOffering(cloudstackTestCase): self.services["service_offering"]["cpuspeed"] = "256" self.services["service_offering"]["memory"] = "128" - serviceOffering_static = ServiceOffering.create(self.apiclient, - self.services["service_offering"]) - + serviceOffering_static = ServiceOffering.create( + self.apiclient, + self.services["service_offering"]) self.services["service_offering"]["cpunumber"] = "" self.services["service_offering"]["cpuspeed"] = "" self.services["service_offering"]["memory"] = "" - serviceOffering_dynamic = ServiceOffering.create(self.apiclient, - self.services["service_offering"]) + serviceOffering_dynamic = ServiceOffering.create( + self.apiclient, + self.services["service_offering"]) self.cleanup_co.append(serviceOffering_static) self.cleanup_co.append(serviceOffering_dynamic) # Deploy VM with static service offering try: - virtualMachine_1 = VirtualMachine.create(apiclient,self.services["virtual_machine"], + virtualMachine_1 = VirtualMachine.create( + apiclient, + self.services["virtual_machine"], serviceofferingid=serviceOffering_static.id, - accountid=self.account.name,domainid=self.account.domainid) + accountid=self.account.name, + domainid=self.account.domainid) except Exception as e: self.fail("vm creation failed: %s" % e) # Scale VM to dynamic service offering try: - virtualMachine_1.scale(apiclient, serviceOfferingId=serviceOffering_dynamic.id, - customcpunumber=4, customcpuspeed=256, custommemory=128) + virtualMachine_1.scale( + apiclient, + serviceOfferingId=serviceOffering_dynamic.id, + customcpunumber=4, + customcpuspeed=256, + custommemory=128) except Exception as e: self.fail("Failure while changing service offering: %s" % e) try: - virtualMachine_2 = VirtualMachine.create(apiclient,self.services["virtual_machine"], + virtualMachine_2 = VirtualMachine.create( + apiclient, + self.services["virtual_machine"], serviceofferingid=serviceOffering_static.id, - accountid=self.account.name,domainid=self.account.domainid) + accountid=self.account.name, + domainid=self.account.domainid) except Exception as e: self.fail("vm creation failed: %s" % e) try: - virtualMachine_2.scale(apiclient, serviceOfferingId=serviceOffering_dynamic.id, - customcpunumber=4) - self.fail("Changing service offering with incomplete data should have failed, it succeded") + virtualMachine_2.scale( + apiclient, + serviceOfferingId=serviceOffering_dynamic.id, + customcpunumber=4) + self.fail( + "Changing service offering with incomplete data should\ + have failed, it succeded") except Exception as e: - self.debug("Failure while changing service offering as expected: %s" % e) + self.debug( + "Failure while changing service offering as expected: %s" % + e) return @data(ADMIN_ACCOUNT, USER_ACCOUNT) - @attr(tags=["basic","advanced"],required_hardware="true") + @attr(tags=["basic", "advanced"],required_hardware="true") def test_change_so_running_vm_dynamic_to_static(self, value): """Test scale running VM from dynamic offering to static offering""" @@ -914,17 +1078,22 @@ class TestScaleVmDynamicServiceOffering(cloudstackTestCase): # 1. Scale operation in step 4 should be successful hypervisor = get_hypervisor_type(self.apiclient) if hypervisor.lower() == "kvm": - self.skipTest("Scaling VM in running state is not supported on KVM") + self.skipTest( + "Scaling VM in running state is not supported on KVM") - isadmin=True + isadmin = True if value == USER_ACCOUNT: - isadmin=False + isadmin = False # Create account and api client - self.account = Account.create(self.apiclient,self.services["account"],domainid=self.domain.id, admin=isadmin) + self.account = Account.create( + self.apiclient, + self.services["account"], + domainid=self.domain.id, + admin=isadmin) apiclient = self.testClient.getUserApiClient( - UserName=self.account.name, - DomainName=self.account.domain) + UserName=self.account.name, + DomainName=self.account.domain) self.cleanup.append(self.account) # Create dynamic and static service offerings @@ -932,38 +1101,47 @@ class TestScaleVmDynamicServiceOffering(cloudstackTestCase): self.services["service_offering"]["cpuspeed"] = "" self.services["service_offering"]["memory"] = "" - serviceOffering_dynamic = ServiceOffering.create(self.apiclient, - self.services["service_offering"]) + serviceOffering_dynamic = ServiceOffering.create( + self.apiclient, + self.services["service_offering"]) self.services["service_offering"]["cpunumber"] = "4" self.services["service_offering"]["cpuspeed"] = "256" self.services["service_offering"]["memory"] = "128" - serviceOffering_static = ServiceOffering.create(self.apiclient, - self.services["service_offering"]) + serviceOffering_static = ServiceOffering.create( + self.apiclient, + self.services["service_offering"]) self.cleanup_co.append(serviceOffering_static) self.cleanup_co.append(serviceOffering_dynamic) # deploy VM with dynamic service offering try: - virtualMachine = VirtualMachine.create(apiclient,self.services["virtual_machine"], + virtualMachine = VirtualMachine.create( + apiclient, + self.services["virtual_machine"], serviceofferingid=serviceOffering_dynamic.id, - accountid=self.account.name,domainid=self.account.domainid, - customcpunumber=2, customcpuspeed=256, custommemory=128) + accountid=self.account.name, + domainid=self.account.domainid, + customcpunumber=2, + customcpuspeed=256, + custommemory=128) except Exception as e: self.fail("vm creation failed: %s" % e) # Scale VM to static service offering try: - virtualMachine.scale(apiclient, serviceOfferingId=serviceOffering_static.id) + virtualMachine.scale( + apiclient, + serviceOfferingId=serviceOffering_static.id) except Exception as e: self.fail("Failure while changing service offering: %s" % e) return @data(ADMIN_ACCOUNT, USER_ACCOUNT) - @attr(tags=["basic","advanced"],required_hardware="true") + @attr(tags=["basic", "advanced"],required_hardware="true") def test_change_so_running_vm_dynamic_to_dynamic(self, value): """Test scale running VM from dynamic offering to dynamic offering""" @@ -983,17 +1161,22 @@ class TestScaleVmDynamicServiceOffering(cloudstackTestCase): hypervisor = get_hypervisor_type(self.apiclient) if hypervisor.lower() == "kvm": - self.skipTest("Scaling VM in running state is not supported on KVM") + self.skipTest( + "Scaling VM in running state is not supported on KVM") - isadmin=True + isadmin = True if value == USER_ACCOUNT: - isadmin=False + isadmin = False # Create account and api client - self.account = Account.create(self.apiclient,self.services["account"],domainid=self.domain.id, admin=isadmin) + self.account = Account.create( + self.apiclient, + self.services["account"], + domainid=self.domain.id, + admin=isadmin) apiclient = self.testClient.getUserApiClient( - UserName=self.account.name, - DomainName=self.account.domain) + UserName=self.account.name, + DomainName=self.account.domain) self.cleanup.append(self.account) # Create dynamic service offerings @@ -1001,56 +1184,78 @@ class TestScaleVmDynamicServiceOffering(cloudstackTestCase): self.services["service_offering"]["cpuspeed"] = "" self.services["service_offering"]["memory"] = "" - serviceOffering_dynamic_1 = ServiceOffering.create(self.apiclient, - self.services["service_offering"]) + serviceOffering_dynamic_1 = ServiceOffering.create( + self.apiclient, + self.services["service_offering"]) - serviceOffering_dynamic_2 = ServiceOffering.create(self.apiclient, - self.services["service_offering"]) + serviceOffering_dynamic_2 = ServiceOffering.create( + self.apiclient, + self.services["service_offering"]) self.cleanup_co.append(serviceOffering_dynamic_1) self.cleanup_co.append(serviceOffering_dynamic_2) # Deploy VM with dynamic service offering try: - virtualMachine = VirtualMachine.create(apiclient,self.services["virtual_machine"], + virtualMachine = VirtualMachine.create( + apiclient, + self.services["virtual_machine"], serviceofferingid=serviceOffering_dynamic_1.id, - accountid=self.account.name,domainid=self.account.domainid, - customcpunumber=2, customcpuspeed=256, custommemory=128) + accountid=self.account.name, + domainid=self.account.domainid, + customcpunumber=2, + customcpuspeed=256, + custommemory=128) except Exception as e: self.fail("vm creation failed: %s" % e) # Scale VM with same dynamic offering try: - virtualMachine.scale(apiclient, serviceOfferingId=serviceOffering_dynamic_1.id, - customcpunumber=4, customcpuspeed=512, custommemory=256) + virtualMachine.scale( + apiclient, + serviceOfferingId=serviceOffering_dynamic_1.id, + customcpunumber=4, + customcpuspeed=512, + custommemory=256) except Exception as e: self.fail("Failure while changing service offering: %s" % e) # Scale VM with other dynamic service offering try: - virtualMachine.scale(apiclient, serviceOfferingId=serviceOffering_dynamic_2.id, - customcpunumber=4, customcpuspeed=512, custommemory=512) + virtualMachine.scale( + apiclient, + serviceOfferingId=serviceOffering_dynamic_2.id, + customcpunumber=4, + customcpuspeed=512, + custommemory=512) except Exception as e: self.fail("Failure while changing service offering: %s" % e) - # Scale VM with dynamic offering proving custom value only for cpu number + # Scale VM with dynamic offering proving custom value only for cpu + # number try: - virtualMachine.scale(apiclient, serviceOfferingId=serviceOffering_dynamic_1.id, - customcpunumber=4) - self.fail("Changing service offering should have failed, it succeded") + virtualMachine.scale( + apiclient, + serviceOfferingId=serviceOffering_dynamic_1.id, + customcpunumber=4) + self.fail( + "Changing service offering should have failed, it succeded") except Exception as e: self.debug("Failure while changing service offering: %s" % e) return + @ddt class TestAccountLimits(cloudstackTestCase): - """Test max limit of account (cpunumber and memory) with dynamic compute offering + + """Test max limit of account (cpunumber and memory) with dynamic + compute offering """ @classmethod def setUpClass(cls): - cloudstackTestClient = super(TestAccountLimits,cls).getClsTestClient() + cloudstackTestClient = super(TestAccountLimits, cls).getClsTestClient() cls.api_client = cloudstackTestClient.getApiClient() # Fill services from the external config file @@ -1058,13 +1263,15 @@ class TestAccountLimits(cloudstackTestCase): # Get Zone, Domain and templates cls.domain = get_domain(cls.api_client) - cls.zone = get_zone(cls.api_client, cloudstackTestClient.getZoneForTests()) + cls.zone = get_zone( + cls.api_client, + cloudstackTestClient.getZoneForTests()) cls.mode = str(cls.zone.networktype).lower() cls.template = get_template( - cls.api_client, - cls.zone.id, - cls.services["ostype"] - ) + cls.api_client, + cls.zone.id, + cls.services["ostype"] + ) cls.services["virtual_machine"]["zoneid"] = cls.zone.id cls.services["virtual_machine"]["template"] = cls.template.id cls._cleanup = [] @@ -1101,9 +1308,10 @@ class TestAccountLimits(cloudstackTestCase): return @data(ADMIN_ACCOUNT, USER_ACCOUNT) - @attr(tags=["basic","advanced"]) + @attr(tags=["basic", "advanced"]) def test_max_account_cpus_deploy_VM(self, value): - """Test cpu limits of account while deploying VM with dynamic compute offering""" + """Test cpu limits of account while deploying VM with dynamic + compute offering""" # Steps: # 1. Create Account (admin/user) @@ -1114,12 +1322,16 @@ class TestAccountLimits(cloudstackTestCase): # Validations: # 1. VM creation should fail - isadmin=True + isadmin = True if value == USER_ACCOUNT: - isadmin=False + isadmin = False # Create account and api client - self.account = Account.create(self.apiclient,self.services["account"],domainid=self.domain.id, admin=isadmin) + self.account = Account.create( + self.apiclient, + self.services["account"], + domainid=self.domain.id, + admin=isadmin) self.cleanup.append(self.account) Resources.updateLimit(self.apiclient, @@ -1133,17 +1345,23 @@ class TestAccountLimits(cloudstackTestCase): self.services["service_offering"]["cpuspeed"] = "" self.services["service_offering"]["memory"] = "" - serviceOffering_dynamic = ServiceOffering.create(self.apiclient, - self.services["service_offering"]) + serviceOffering_dynamic = ServiceOffering.create( + self.apiclient, + self.services["service_offering"]) self.cleanup_co.append(serviceOffering_dynamic) # Deploy VM with dynamic service offering try: - VirtualMachine.create(self.apiclient,self.services["virtual_machine"], + VirtualMachine.create( + self.apiclient, + self.services["virtual_machine"], serviceofferingid=serviceOffering_dynamic.id, - accountid=self.account.name,domainid=self.account.domainid, - customcpunumber=3, customcpuspeed=256, custommemory=128) + accountid=self.account.name, + domainid=self.account.domainid, + customcpunumber=3, + customcpuspeed=256, + custommemory=128) self.fail("vm creation should have failed, it succeeded") except Exception as e: self.debug("vm creation failed as expected with error: %s" % e) @@ -1151,9 +1369,10 @@ class TestAccountLimits(cloudstackTestCase): return @data(ADMIN_ACCOUNT, USER_ACCOUNT) - @attr(tags=["basic","advanced"]) + @attr(tags=["basic", "advanced"]) def test_max_account_cpus_scale_VM(self, value): - """Test cpu limits of account while scaling VM with dynamic compute offering""" + """Test cpu limits of account while scaling VM with dynamic + compute offering""" # Steps: # 1. Create Account (admin/user) @@ -1166,12 +1385,16 @@ class TestAccountLimits(cloudstackTestCase): # 1. VM creation should succeed # 2. VM scaling operation should fail - isadmin=True + isadmin = True if value == USER_ACCOUNT: - isadmin=False + isadmin = False # Create account and api client - self.account = Account.create(self.apiclient,self.services["account"],domainid=self.domain.id, admin=isadmin) + self.account = Account.create( + self.apiclient, + self.services["account"], + domainid=self.domain.id, + admin=isadmin) self.cleanup.append(self.account) Resources.updateLimit(self.apiclient, @@ -1185,35 +1408,48 @@ class TestAccountLimits(cloudstackTestCase): self.services["service_offering"]["cpuspeed"] = "" self.services["service_offering"]["memory"] = "" - serviceOffering_dynamic = ServiceOffering.create(self.apiclient, - self.services["service_offering"]) + serviceOffering_dynamic = ServiceOffering.create( + self.apiclient, + self.services["service_offering"]) self.cleanup_co.append(serviceOffering_dynamic) # Deploy VM with dynamic service offering try: - virtualMachine = VirtualMachine.create(self.apiclient,self.services["virtual_machine"], + virtualMachine = VirtualMachine.create( + self.apiclient, + self.services["virtual_machine"], serviceofferingid=serviceOffering_dynamic.id, - accountid=self.account.name,domainid=self.account.domainid, - customcpunumber=2, customcpuspeed=256, custommemory=128) + accountid=self.account.name, + domainid=self.account.domainid, + customcpunumber=2, + customcpuspeed=256, + custommemory=128) except Exception as e: self.fail("vm creation failed: %s" % e) # Scale VM with same dynamic offering try: - virtualMachine.scale(self.apiclient, serviceOfferingId=serviceOffering_dynamic.id, - customcpunumber=4, customcpuspeed=512, custommemory=256) + virtualMachine.scale( + self.apiclient, + serviceOfferingId=serviceOffering_dynamic.id, + customcpunumber=4, + customcpuspeed=512, + custommemory=256) self.fail("Scaling virtual machine with cpu number more than \ allowed limit (of account) succeded, should have failed") except Exception as e: - self.debug("Failure while changing service offering as expected: %s" % e) + self.debug( + "Failure while changing service offering as expected: %s" % + e) return @data(ADMIN_ACCOUNT, USER_ACCOUNT) - @attr(tags=["basic","advanced"]) + @attr(tags=["basic", "advanced"]) def test_max_account_memory_deploy_VM(self, value): - """Test memory limits of account while deploying VM with dynamic compute offering""" + """Test memory limits of account while deploying VM with dynamic + compute offering""" # Steps: # 1. Create Account (admin/user) @@ -1224,12 +1460,16 @@ class TestAccountLimits(cloudstackTestCase): # Validations: # 1. VM creation should fail - isadmin=True + isadmin = True if value == USER_ACCOUNT: - isadmin=False + isadmin = False # Create account and api client - self.account = Account.create(self.apiclient,self.services["account"],domainid=self.domain.id, admin=isadmin) + self.account = Account.create( + self.apiclient, + self.services["account"], + domainid=self.domain.id, + admin=isadmin) self.cleanup.append(self.account) Resources.updateLimit(self.apiclient, @@ -1243,17 +1483,23 @@ class TestAccountLimits(cloudstackTestCase): self.services["service_offering"]["cpuspeed"] = "" self.services["service_offering"]["memory"] = "" - serviceOffering_dynamic = ServiceOffering.create(self.apiclient, - self.services["service_offering"]) + serviceOffering_dynamic = ServiceOffering.create( + self.apiclient, + self.services["service_offering"]) self.cleanup_co.append(serviceOffering_dynamic) # Deploy VM with dynamic service offering try: - VirtualMachine.create(self.apiclient,self.services["virtual_machine"], + VirtualMachine.create( + self.apiclient, + self.services["virtual_machine"], serviceofferingid=serviceOffering_dynamic.id, - accountid=self.account.name,domainid=self.account.domainid, - customcpunumber=3, customcpuspeed=256, custommemory=512) + accountid=self.account.name, + domainid=self.account.domainid, + customcpunumber=3, + customcpuspeed=256, + custommemory=512) self.fail("vm creation should have failed, it succeeded") except Exception as e: self.debug("vm creation failed as expected with error: %s" % e) @@ -1261,9 +1507,10 @@ class TestAccountLimits(cloudstackTestCase): return @data(ADMIN_ACCOUNT, USER_ACCOUNT) - @attr(tags=["basic","advanced"]) + @attr(tags=["basic", "advanced"]) def test_max_account_memory_scale_VM(self, value): - """Test memory limits of account while scaling VM with dynamic compute offering""" + """Test memory limits of account while scaling VM with + dynamic compute offering""" # Steps: # 1. Create Account (admin/user) @@ -1276,12 +1523,16 @@ class TestAccountLimits(cloudstackTestCase): # 1. VM creation should succeed # 2. VM scaling operation should fail - isadmin=True + isadmin = True if value == USER_ACCOUNT: - isadmin=False + isadmin = False # Create account and api client - self.account = Account.create(self.apiclient,self.services["account"],domainid=self.domain.id, admin=isadmin) + self.account = Account.create( + self.apiclient, + self.services["account"], + domainid=self.domain.id, + admin=isadmin) self.cleanup.append(self.account) Resources.updateLimit(self.apiclient, @@ -1295,39 +1546,53 @@ class TestAccountLimits(cloudstackTestCase): self.services["service_offering"]["cpuspeed"] = "" self.services["service_offering"]["memory"] = "" - serviceOffering_dynamic = ServiceOffering.create(self.apiclient, - self.services["service_offering"]) + serviceOffering_dynamic = ServiceOffering.create( + self.apiclient, + self.services["service_offering"]) self.cleanup_co.append(serviceOffering_dynamic) # Deploy VM with dynamic service offering try: - virtualMachine = VirtualMachine.create(self.apiclient,self.services["virtual_machine"], + virtualMachine = VirtualMachine.create( + self.apiclient, + self.services["virtual_machine"], serviceofferingid=serviceOffering_dynamic.id, - accountid=self.account.name,domainid=self.account.domainid, - customcpunumber=2, customcpuspeed=256, custommemory=256) + accountid=self.account.name, + domainid=self.account.domainid, + customcpunumber=2, + customcpuspeed=256, + custommemory=256) except Exception as e: self.fail("vm creation failed: %s" % e) # Scale VM with same dynamic offering try: - virtualMachine.scale(self.apiclient, serviceOfferingId=serviceOffering_dynamic.id, - customcpunumber=4, customcpuspeed=512, custommemory=512) + virtualMachine.scale( + self.apiclient, + serviceOfferingId=serviceOffering_dynamic.id, + customcpunumber=4, + customcpuspeed=512, + custommemory=512) self.fail("Scaling virtual machine with cpu number more than \ allowed limit (of account) succeded, should have failed") except Exception as e: - self.debug("Failure while changing service offering as expected: %s" % e) + self.debug( + "Failure while changing service offering as expected: %s" % + e) return + @ddt class TestAffinityGroup(cloudstackTestCase): + """Test affinity group working with VMs created with dynamic offering """ @classmethod def setUpClass(cls): - cloudstackTestClient = super(TestAffinityGroup,cls).getClsTestClient() + cloudstackTestClient = super(TestAffinityGroup, cls).getClsTestClient() cls.api_client = cloudstackTestClient.getApiClient() # Fill services from the external config file @@ -1335,13 +1600,15 @@ class TestAffinityGroup(cloudstackTestCase): # Get Zone, Domain and templates cls.domain = get_domain(cls.api_client) - cls.zone = get_zone(cls.api_client, cloudstackTestClient.getZoneForTests()) + cls.zone = get_zone( + cls.api_client, + cloudstackTestClient.getZoneForTests()) cls.mode = str(cls.zone.networktype).lower() cls.template = get_template( - cls.api_client, - cls.zone.id, - cls.services["ostype"] - ) + cls.api_client, + cls.zone.id, + cls.services["ostype"] + ) cls.services["virtual_machine"]["zoneid"] = cls.zone.id cls.services["virtual_machine"]["template"] = cls.template.id cls._cleanup = [] @@ -1378,7 +1645,7 @@ class TestAffinityGroup(cloudstackTestCase): return @data(ADMIN_ACCOUNT, USER_ACCOUNT) - @attr(tags=["basic","advanced"], BugId="7180", required_hardware="true") + @attr(tags=["basic", "advanced"], BugId="7180", required_hardware="true") def test_deploy_VM_with_affinity_group(self, value): """Test deploy VMs with affinity group and dynamic compute offering""" @@ -1391,12 +1658,16 @@ class TestAffinityGroup(cloudstackTestCase): # Validations: # 1. VM creation should fail - isadmin=True + isadmin = True if value == USER_ACCOUNT: - isadmin=False + isadmin = False # Create account and api client - self.account = Account.create(self.apiclient,self.services["account"],domainid=self.domain.id, admin=isadmin) + self.account = Account.create( + self.apiclient, + self.services["account"], + domainid=self.domain.id, + admin=isadmin) self.cleanup.append(self.account) # Create dynamic service offerings @@ -1404,43 +1675,68 @@ class TestAffinityGroup(cloudstackTestCase): self.services["service_offering"]["cpuspeed"] = "" self.services["service_offering"]["memory"] = "" - serviceOffering_dynamic = ServiceOffering.create(self.apiclient, - self.services["service_offering"]) + serviceOffering_dynamic = ServiceOffering.create( + self.apiclient, + self.services["service_offering"]) self.cleanup_co.append(serviceOffering_dynamic) - self.services["host_anti_affinity"]["name"] = "aff_grp_" + random_gen(size=6) - affinityGroup = AffinityGroup.create(self.apiclient, self.services["host_anti_affinity"], - self.account.name, self.domain.id) + self.services["host_anti_affinity"][ + "name"] = "aff_grp_" + random_gen(size=6) + affinityGroup = AffinityGroup.create( + self.apiclient, + self.services["host_anti_affinity"], + self.account.name, + self.domain.id) # Deploy VM with dynamic service offering try: - virtualMachine = VirtualMachine.create(self.apiclient,self.services["virtual_machine"], + virtualMachine = VirtualMachine.create( + self.apiclient, + self.services["virtual_machine"], serviceofferingid=serviceOffering_dynamic.id, - accountid=self.account.name,domainid=self.account.domainid, - customcpunumber=2, customcpuspeed=256, custommemory=128, - affinitygroupnames=[affinityGroup.name]) + accountid=self.account.name, + domainid=self.account.domainid, + customcpunumber=2, + customcpuspeed=256, + custommemory=128, + affinitygroupnames=[ + affinityGroup.name]) except Exception as e: self.fail("vm creation failed with error: %s" % e) - otherHostsInCluster = Host.list(self.apiclient, virtualmachineid=virtualMachine.id) + otherHostsInCluster = Host.list( + self.apiclient, + virtualmachineid=virtualMachine.id) if validateList(otherHostsInCluster)[0] == PASS: try: - VirtualMachine.create(self.apiclient,self.services["virtual_machine"], - serviceofferingid=serviceOffering_dynamic.id, - accountid=self.account.name,domainid=self.account.domainid, - customcpunumber=2, customcpuspeed=256, custommemory=128, - affinitygroupnames=[affinityGroup.name]) + VirtualMachine.create( + self.apiclient, + self.services["virtual_machine"], + serviceofferingid=serviceOffering_dynamic.id, + accountid=self.account.name, + domainid=self.account.domainid, + customcpunumber=2, + customcpuspeed=256, + custommemory=128, + affinitygroupnames=[ + affinityGroup.name]) except Exception as e: self.fail("vm creation failed with error: %s" % e) else: try: - VirtualMachine.create(self.apiclient,self.services["virtual_machine"], - serviceofferingid=serviceOffering_dynamic.id, - accountid=self.account.name,domainid=self.account.domainid, - customcpunumber=2, customcpuspeed=256, custommemory=128, - affinitygroupnames=[affinityGroup.name]) + VirtualMachine.create( + self.apiclient, + self.services["virtual_machine"], + serviceofferingid=serviceOffering_dynamic.id, + accountid=self.account.name, + domainid=self.account.domainid, + customcpunumber=2, + customcpuspeed=256, + custommemory=128, + affinitygroupnames=[ + affinityGroup.name]) self.fail("vm creation should have failed, it succeded") except Exception as e: self.debug("vm creation failed as expected with error: %s" % e) From 239150c558380ee860ae991c0d7f197030087463 Mon Sep 17 00:00:00 2001 From: Gaurav Aradhye Date: Tue, 23 Dec 2014 19:02:56 +0530 Subject: [PATCH 023/391] CLOUDSTACK-8117: Increase the allowed margin (+/-) for memory of VM on hyperv used to equate with the memory specified in service offering Signed-off-by: SrikanteswaraRao Talluri --- .../smoke/test_service_offerings.py | 157 +++++++++++------- 1 file changed, 100 insertions(+), 57 deletions(-) diff --git a/test/integration/smoke/test_service_offerings.py b/test/integration/smoke/test_service_offerings.py index e390f75572e..1b1b113e8ea 100644 --- a/test/integration/smoke/test_service_offerings.py +++ b/test/integration/smoke/test_service_offerings.py @@ -5,9 +5,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -16,21 +16,22 @@ # under the License. """ BVT tests for Service offerings""" -#Import Local Modules +# Import Local Modules from marvin.codes import FAILED from marvin.cloudstackTestCase import cloudstackTestCase -from marvin.cloudstackAPI import changeServiceForVirtualMachine,updateServiceOffering +from marvin.cloudstackAPI import (changeServiceForVirtualMachine, + updateServiceOffering) from marvin.lib.utils import (isAlmostEqual, - cleanup_resources, - random_gen) + cleanup_resources, + random_gen) from marvin.lib.base import (ServiceOffering, - Account, - VirtualMachine) + Account, + VirtualMachine) from marvin.lib.common import (list_service_offering, - list_virtual_machines, - get_domain, - get_zone, - get_template) + list_virtual_machines, + get_domain, + get_zone, + get_template) from nose.plugins.attrib import attr @@ -38,6 +39,7 @@ _multiprocess_shared_ = True class TestCreateServiceOffering(cloudstackTestCase): + def setUp(self): self.apiclient = self.testClient.getApiClient() self.dbclient = self.testClient.getDbConnection() @@ -46,7 +48,7 @@ class TestCreateServiceOffering(cloudstackTestCase): def tearDown(self): try: - #Clean up, terminate the created templates + # Clean up, terminate the created templates cleanup_resources(self.apiclient, self.cleanup) except Exception as e: @@ -54,12 +56,21 @@ class TestCreateServiceOffering(cloudstackTestCase): return - @attr(tags=["advanced", "advancedns", "smoke", "basic", "eip", "sg"], required_hardware="false") + @attr( + tags=[ + "advanced", + "advancedns", + "smoke", + "basic", + "eip", + "sg"], + required_hardware="false") def test_01_create_service_offering(self): """Test to create service offering""" # Validate the following: - # 1. createServiceOfferings should return a valid information for newly created offering + # 1. createServiceOfferings should return a valid information + # for newly created offering # 2. The Cloud Database contains the valid information service_offering = ServiceOffering.create( @@ -68,7 +79,9 @@ class TestCreateServiceOffering(cloudstackTestCase): ) self.cleanup.append(service_offering) - self.debug("Created service offering with ID: %s" % service_offering.id) + self.debug( + "Created service offering with ID: %s" % + service_offering.id) list_service_response = list_service_offering( self.apiclient, @@ -115,6 +128,7 @@ class TestCreateServiceOffering(cloudstackTestCase): class TestServiceOfferings(cloudstackTestCase): + def setUp(self): self.apiclient = self.testClient.getApiClient() self.dbclient = self.testClient.getDbConnection() @@ -122,7 +136,7 @@ class TestServiceOfferings(cloudstackTestCase): def tearDown(self): try: - #Clean up, terminate the created templates + # Clean up, terminate the created templates cleanup_resources(self.apiclient, self.cleanup) except Exception as e: @@ -135,6 +149,7 @@ class TestServiceOfferings(cloudstackTestCase): testClient = super(TestServiceOfferings, cls).getClsTestClient() cls.apiclient = testClient.getApiClient() cls.services = testClient.getParsedTestDataConfig() + cls.hypervisor = testClient.getHypervisorInfo() domain = get_domain(cls.apiclient) cls.zone = get_zone(cls.apiclient, testClient.getZoneForTests()) @@ -149,12 +164,13 @@ class TestServiceOfferings(cloudstackTestCase): cls.services["service_offerings"] ) template = get_template( - cls.apiclient, - cls.zone.id, - cls.services["ostype"] - ) + cls.apiclient, + cls.zone.id, + cls.services["ostype"] + ) if template == FAILED: - assert False, "get_template() failed to return template with description %s" % cls.services["ostype"] + assert False, "get_template() failed to return\ + template with description %s" % cls.services["ostype"] # Set Zones and disk offerings cls.services["small"]["zoneid"] = cls.zone.id @@ -165,47 +181,57 @@ class TestServiceOfferings(cloudstackTestCase): # Create VMs, NAT Rules etc cls.account = Account.create( - cls.apiclient, - cls.services["account"], - domainid=domain.id - ) + cls.apiclient, + cls.services["account"], + domainid=domain.id + ) cls.small_offering = ServiceOffering.create( - cls.apiclient, - cls.services["service_offerings"]["small"] - ) + cls.apiclient, + cls.services["service_offerings"]["small"] + ) cls.medium_offering = ServiceOffering.create( - cls.apiclient, - cls.services["service_offerings"]["medium"] - ) + cls.apiclient, + cls.services["service_offerings"]["medium"] + ) cls.medium_virtual_machine = VirtualMachine.create( - cls.apiclient, - cls.services["medium"], - accountid=cls.account.name, - domainid=cls.account.domainid, - serviceofferingid=cls.medium_offering.id, - mode=cls.services["mode"] - ) + cls.apiclient, + cls.services["medium"], + accountid=cls.account.name, + domainid=cls.account.domainid, + serviceofferingid=cls.medium_offering.id, + mode=cls.services["mode"] + ) cls._cleanup = [ - cls.small_offering, - cls.medium_offering, - cls.account - ] + cls.small_offering, + cls.medium_offering, + cls.account + ] return @classmethod def tearDownClass(cls): try: - cls.apiclient = super(TestServiceOfferings, cls).getClsTestClient().getApiClient() - #Clean up, terminate the created templates + cls.apiclient = super( + TestServiceOfferings, + cls).getClsTestClient().getApiClient() + # Clean up, terminate the created templates cleanup_resources(cls.apiclient, cls._cleanup) except Exception as e: raise Exception("Warning: Exception during cleanup : %s" % e) return - @attr(tags=["advanced", "advancedns", "smoke", "basic", "eip", "sg"], required_hardware="false") + @attr( + tags=[ + "advanced", + "advancedns", + "smoke", + "basic", + "eip", + "sg"], + required_hardware="false") def test_02_edit_service_offering(self): """Test to update existing service offering""" @@ -213,7 +239,7 @@ class TestServiceOfferings(cloudstackTestCase): # 1. updateServiceOffering should return # a valid information for newly created offering - #Generate new name & displaytext from random data + # Generate new name & displaytext from random data random_displaytext = random_gen() random_name = random_gen() @@ -221,7 +247,7 @@ class TestServiceOfferings(cloudstackTestCase): self.service_offering_1.id) cmd = updateServiceOffering.updateServiceOfferingCmd() - #Add parameters for API call + # Add parameters for API call cmd.id = self.service_offering_1.id cmd.displaytext = random_displaytext cmd.name = random_name @@ -256,7 +282,15 @@ class TestServiceOfferings(cloudstackTestCase): return - @attr(tags=["advanced", "advancedns", "smoke", "basic", "eip", "sg"], required_hardware="false") + @attr( + tags=[ + "advanced", + "advancedns", + "smoke", + "basic", + "eip", + "sg"], + required_hardware="false") def test_03_delete_service_offering(self): """Test to delete service offering""" @@ -316,22 +350,23 @@ class TestServiceOfferings(cloudstackTestCase): self.debug("VM state: %s" % vm.state) else: raise Exception( - "Failed to start VM (ID: %s) after changing service offering" % vm.id) + "Failed to start VM (ID: %s) after changing\ + service offering" % vm.id) try: ssh = self.medium_virtual_machine.get_ssh_client() except Exception as e: self.fail( - "SSH Access failed for %s: %s" %\ + "SSH Access failed for %s: %s" % (self.medium_virtual_machine.ipaddress, e) ) cpuinfo = ssh.execute("cat /proc/cpuinfo") cpu_cnt = len([i for i in cpuinfo if "processor" in i]) - #'cpu MHz\t\t: 2660.499' + # 'cpu MHz\t\t: 2660.499' cpu_speed = [i for i in cpuinfo if "cpu MHz" in i][0].split()[3] meminfo = ssh.execute("cat /proc/meminfo") - #MemTotal: 1017464 kB + # MemTotal: 1017464 kB total_mem = [i for i in meminfo if "MemTotal" in i][0].split()[1] self.debug( @@ -339,7 +374,7 @@ class TestServiceOfferings(cloudstackTestCase): cpu_cnt, cpu_speed, total_mem - )) + )) self.assertAlmostEqual( int(cpu_cnt), self.small_offering.cpunumber, @@ -350,11 +385,19 @@ class TestServiceOfferings(cloudstackTestCase): self.small_offering.cpuspeed, "Check CPU Speed for small offering" ) + + range = 20 + if self.hypervisor.lower() == "hyperv": + range = 200 + # TODO: Find the memory allocated to VM on hyperv hypervisor using + # powershell commands and use that value to equate instead of + # manipulating range, currently we get the memory count much less + # because of the UI component self.assertTrue( isAlmostEqual(int(int(total_mem) / 1024), - int(self.small_offering.memory), - range=20 - ), + int(self.small_offering.memory), + range=range + ), "Check Memory(kb) for small offering" ) return From 94237a4c5b08356bb0a4950ce9fd7ec78f2168f9 Mon Sep 17 00:00:00 2001 From: Gaurav Aradhye Date: Tue, 23 Dec 2014 17:29:31 +0530 Subject: [PATCH 024/391] CLOUDSTACK-8116: Moved ldap data to configurableData section in test_data.py and made related changes in the test case Signed-off-by: SrikanteswaraRao Talluri --- test/integration/component/test_ldap.py | 122 ++++++++++++++---------- tools/marvin/marvin/config/test_data.py | 35 ++++--- 2 files changed, 88 insertions(+), 69 deletions(-) diff --git a/test/integration/component/test_ldap.py b/test/integration/component/test_ldap.py index 5a789570b66..34640227215 100644 --- a/test/integration/component/test_ldap.py +++ b/test/integration/component/test_ldap.py @@ -20,22 +20,19 @@ #!/usr/bin/env python - -import marvin -from marvin import cloudstackTestCase -from marvin.cloudstackTestCase import * -import unittest -import hashlib -import random -from marvin.cloudstackAPI import * +from marvin.cloudstackTestCase import cloudstackTestCase +from marvin.cloudstackAPI import (updateConfiguration, + createAccount, + deleteAccount, + addLdapConfiguration, + deleteLdapConfiguration) from marvin.cloudstackAPI import login -from marvin.lib.utils import * -from marvin.lib.base import * -from marvin.lib.common import * +from marvin.lib.utils import cleanup_resources from nose.plugins.attrib import attr -import urllib + class TestLdap(cloudstackTestCase): + """ This tests attempts to register a LDAP server and authenticate as an LDAP user. """ @@ -46,11 +43,8 @@ class TestLdap(cloudstackTestCase): testClient = super(TestLdap, cls).getClsTestClient() cls.api_client = testClient.getApiClient() cls.services = testClient.getParsedTestDataConfig() - cls.account = cls.services["ldap_account"] cls._cleanup = [] - - @classmethod def tearDownClass(cls): try: @@ -66,12 +60,18 @@ class TestLdap(cloudstackTestCase): self.acct = createAccount.createAccountCmd() self.acct.accounttype = 0 - self.acct.firstname = self.services["ldap_account"]["firstname"] - self.acct.lastname = self.services["ldap_account"]["lastname"] - self.acct.password = self.services["ldap_account"]["password"] - self.acct.username = self.services["ldap_account"]["username"] - self.acct.email = self.services["ldap_account"]["email"] - self.acct.account = self.services["ldap_account"]["username"] + self.acct.firstname = self.services[ + "configurableData"]["ldap_account"]["firstname"] + self.acct.lastname = self.services[ + "configurableData"]["ldap_account"]["lastname"] + self.acct.password = self.services[ + "configurableData"]["ldap_account"]["password"] + self.acct.username = self.services[ + "configurableData"]["ldap_account"]["username"] + self.acct.email = self.services[ + "configurableData"]["ldap_account"]["email"] + self.acct.account = self.services[ + "configurableData"]["ldap_account"]["username"] self.acct.domainid = 1 self.acctRes = self.apiClient.createAccount(self.acct) @@ -84,14 +84,17 @@ class TestLdap(cloudstackTestCase): deleteAcct = deleteAccount.deleteAccountCmd() deleteAcct.id = self.acctRes.id - acct_name=self.acctRes.name + acct_name = self.acctRes.name self.apiClient.deleteAccount(deleteAcct) - self.debug("Deleted the the following account name %s:" %acct_name) + self.debug( + "Deleted the the following account name %s:" % + acct_name) - if(self.ldapconfRes==1): - self._deleteLdapConfiguration(self.services["ldapConfiguration_1"]) + if(self.ldapconfRes == 1): + self._deleteLdapConfiguration( + self.services["configurableData"]["ldap_configuration"]) except Exception as e: raise Exception("Warning: Exception during cleanup : %s" % e) @@ -103,30 +106,33 @@ class TestLdap(cloudstackTestCase): This test configures LDAP and attempts to authenticate as a user. """ - self.debug("start test") - self.ldapconfRes=self._addLdapConfiguration(self.services["ldapConfiguration_1"]) + self.ldapconfRes = self._addLdapConfiguration( + self.services["configurableData"]["ldap_configuration"]) - if(self.ldapconfRes==1): + if(self.ldapconfRes == 1): self.debug("Ldap Configuration was succcessful") - loginRes = self._checkLogin(self.services["ldapConfiguration_1"]["ldapUsername"],self.services["ldapConfiguration_1"]["ldapPassword"]) + loginRes = self._checkLogin( + self.services["configurableData"]["ldap_configuration"]["ldapUsername"], + self.services["configurableData"]["ldap_configuration"]["ldapPassword"]) self.debug(loginRes) - self.assertEquals(loginRes,1,"Ldap Authentication") + self.assertEquals(loginRes, 1, "Ldap Authentication") else: self.debug("LDAP Configuration failed with exception") - self.assertEquals(self.ldapconfRes,1,"addLdapConfiguration failed") - + self.assertEquals( + self.ldapconfRes, + 1, + "addLdapConfiguration failed") self.debug("end test") - def _addLdapConfiguration(self,ldapConfiguration): - + def _addLdapConfiguration(self, ldapConfiguration): """ :param ldapConfiguration @@ -138,27 +144,42 @@ class TestLdap(cloudstackTestCase): updateConfigurationCmd = updateConfiguration.updateConfigurationCmd() updateConfigurationCmd.name = "ldap.basedn" updateConfigurationCmd.value = ldapConfiguration['basedn'] - updateConfigurationResponse = self.apiClient.updateConfiguration(updateConfigurationCmd) - self.debug("updated the parameter %s with value %s"%(updateConfigurationResponse.name, updateConfigurationResponse.value)) + updateConfigurationResponse = self.apiClient.updateConfiguration( + updateConfigurationCmd) + self.debug( + "updated the parameter %s with value %s" % + (updateConfigurationResponse.name, + updateConfigurationResponse.value)) updateConfigurationCmd = updateConfiguration.updateConfigurationCmd() updateConfigurationCmd.name = "ldap.email.attribute" updateConfigurationCmd.value = ldapConfiguration['emailAttribute'] - updateConfigurationResponse = self.apiClient.updateConfiguration(updateConfigurationCmd) - self.debug("updated the parameter %s with value %s"%(updateConfigurationResponse.name, updateConfigurationResponse.value)) + updateConfigurationResponse = self.apiClient.updateConfiguration( + updateConfigurationCmd) + self.debug( + "updated the parameter %s with value %s" % + (updateConfigurationResponse.name, + updateConfigurationResponse.value)) updateConfigurationCmd = updateConfiguration.updateConfigurationCmd() updateConfigurationCmd.name = "ldap.user.object" updateConfigurationCmd.value = ldapConfiguration['userObject'] - updateConfigurationResponse = self.apiClient.updateConfiguration(updateConfigurationCmd) - self.debug("updated the parameter %s with value %s"%(updateConfigurationResponse.name, updateConfigurationResponse.value)) - + updateConfigurationResponse = self.apiClient.updateConfiguration( + updateConfigurationCmd) + self.debug( + "updated the parameter %s with value %s" % + (updateConfigurationResponse.name, + updateConfigurationResponse.value)) updateConfigurationCmd = updateConfiguration.updateConfigurationCmd() updateConfigurationCmd.name = "ldap.username.attribute" updateConfigurationCmd.value = ldapConfiguration['usernameAttribute'] - updateConfigurationResponse = self.apiClient.updateConfiguration(updateConfigurationCmd) - self.debug("updated the parameter %s with value %s"%(updateConfigurationResponse.name, updateConfigurationResponse.value)) + updateConfigurationResponse = self.apiClient.updateConfiguration( + updateConfigurationCmd) + self.debug( + "updated the parameter %s with value %s" % + (updateConfigurationResponse.name, + updateConfigurationResponse.value)) self.debug("start addLdapConfiguration test") @@ -171,12 +192,11 @@ class TestLdap(cloudstackTestCase): self.apiClient.addLdapConfiguration(ldapServer) self.debug("addLdapConfiguration was successful") return 1 - except Exception, e: - self.debug("addLdapConfiguration failed %s" %e) + except Exception as e: + self.debug("addLdapConfiguration failed %s" % e) return 0 - def _deleteLdapConfiguration(self,ldapConfiguration): - + def _deleteLdapConfiguration(self, ldapConfiguration): """ :param ldapConfiguration @@ -190,8 +210,8 @@ class TestLdap(cloudstackTestCase): self.apiClient.deleteLdapConfiguration(ldapServer) self.debug("deleteLdapConfiguration was successful") return 1 - except Exception, e: - self.debug("deleteLdapConfiguration failed %s" %e) + except Exception as e: + self.debug("deleteLdapConfiguration failed %s" % e) return 0 def _checkLogin(self, username, password): @@ -216,6 +236,6 @@ class TestLdap(cloudstackTestCase): self.debug("login successful") return 1 - except Exception, p: - self.debug("login operation failed %s" %p) + except Exception as p: + self.debug("login operation failed %s" % p) self.debug("end of Login") diff --git a/tools/marvin/marvin/config/test_data.py b/tools/marvin/marvin/config/test_data.py index ba2fb24840e..75609dd1e47 100644 --- a/tools/marvin/marvin/config/test_data.py +++ b/tools/marvin/marvin/config/test_data.py @@ -1383,24 +1383,6 @@ test_data = { }, "ostype": 'CentOS 5.6 (64-bit)', }, - "ldap_account": { - "email": "rmurphy@cloudstack.org", - "firstname": "Ryan", - "lastname": "Murphy", - "username": "rmurphy", - "password": "internalcloudstackpassword", - }, - "ldapConfiguration_1": { - "basedn": "dc=cloudstack,dc=org", - "emailAttribute": "mail", - "userObject": "inetOrgPerson", - "usernameAttribute": "uid", - "hostname": "localhost", - "port": "10389", - "ldapUsername": "rmurphy", - "ldapPassword": "password" - }, - "test_34_DeployVM_in_SecondSGNetwork": { "zone": "advsg", "config": "D:\ACS-Repo\setup\dev\\advancedsg.cfg",#Absolute path to cfg file @@ -1461,6 +1443,23 @@ test_data = { "username": "root", "password": "password", }, + "ldap_account": { + "email": "", + "firstname": "", + "lastname": "", + "username": "", + "password": "", + }, + "ldap_configuration": { + "basedn": "", + "emailAttribute": "", + "userObject": "", + "usernameAttribute": "", + "hostname": "", + "port": "", + "ldapUsername": "", + "ldapPassword": "" + }, "systemVmDelay": 120 } } From 85e88e9cbcba913fd9848e1daa42e21259c02a96 Mon Sep 17 00:00:00 2001 From: Jessica Wang Date: Wed, 31 Dec 2014 16:21:13 -0800 Subject: [PATCH 025/391] CLOUDSTACK-8139: UI > create compute offering > server-side only supports one single host tag instead of multiple host tags. So, change UI to take in only one single host tag instead of multiple host tags in create compute dialog. --- ui/scripts/configuration.js | 42 +++++-------------------------------- 1 file changed, 5 insertions(+), 37 deletions(-) diff --git a/ui/scripts/configuration.js b/ui/scripts/configuration.js index 6468d223fd6..da01b4b27c7 100644 --- a/ui/scripts/configuration.js +++ b/ui/scripts/configuration.js @@ -339,42 +339,10 @@ }); } }, - hostTags: { - label: 'label.host.tags', - docID: 'helpComputeOfferingHostTags', - isTokenInput: true, - dataProvider: function(args) { - $.ajax({ - url: createURL("listHostTags"), - dataType: "json", - success: function(json) { - var item = json.listhosttagsresponse.hosttag; - var tags = []; - - if (item != null) - { - tags = $.map(item, function(tag) { - return { - id: tag.name, - name: tag.name - }; - }); - } - - args.response.success({ - data: tags, - hintText: "Type in part of a host tag", - noResultsText: "No host tags found" - }); - }, - error: function(XMLHttpResponse) { - var errorMsg = parseXMLHttpResponse(XMLHttpResponse); - - args.response.error(errorMsg); - } - }); - } - }, + hostTags: { //Only one single host tag is supported at server-side. Multiple host tags are NOT supported at server-side. + label: 'Host Tag', + docID: 'helpComputeOfferingHostTags' + }, cpuCap: { label: 'label.CPU.cap', isBoolean: true, @@ -958,7 +926,7 @@ label: 'label.storage.tags' }, hosttags: { - label: 'label.host.tags' + label: 'Host Tag' }, domain: { label: 'label.domain' From 775e4d34c3236ca58749b4b6065d6b8f0b6ce555 Mon Sep 17 00:00:00 2001 From: Jessica Wang Date: Fri, 2 Jan 2015 11:33:57 -0800 Subject: [PATCH 026/391] CLOUDSTACK-8141: UI > use Project view > Infrastructure > zone > physical network > Public traffic type > do not pass projectId to listNetworks API. --- ui/scripts/system.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ui/scripts/system.js b/ui/scripts/system.js index 005b47e1b12..fdaaed4bdee 100644 --- a/ui/scripts/system.js +++ b/ui/scripts/system.js @@ -601,7 +601,9 @@ dataProvider: function (args) { $.ajax({ - url: createURL("listNetworks&listAll=true&trafficType=Public&isSystem=true&zoneId=" + selectedZoneObj.id), + url: createURL("listNetworks&listAll=true&trafficType=Public&isSystem=true&zoneId=" + selectedZoneObj.id, { + ignoreProject: true + }), dataType: "json", async: false, success: function (json) { From 744c1a1b03c5485dc51c4b85c7c22052f6e23934 Mon Sep 17 00:00:00 2001 From: Jessica Wang Date: Fri, 2 Jan 2015 14:06:28 -0800 Subject: [PATCH 027/391] CLOUDSTACK-7383: UI > Instances menu > detail view > TakeSnapshot action should be hidden when VM's hypervisor is LXC. --- ui/scripts/instances.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui/scripts/instances.js b/ui/scripts/instances.js index 26748048f20..4ff1bff8236 100644 --- a/ui/scripts/instances.js +++ b/ui/scripts/instances.js @@ -2265,7 +2265,7 @@ allowedActions.push("restart"); if ((jsonObj.hypervisor != 'KVM' || g_kvmsnapshotenabled == true) - || (jsonObj.hypervisor != 'LXC')) { + && (jsonObj.hypervisor != 'LXC')) { allowedActions.push("snapshot"); } @@ -2299,7 +2299,7 @@ allowedActions.push("reinstall"); if ((jsonObj.hypervisor != 'KVM' || g_kvmsnapshotenabled == true) - || (jsonObj.hypervisor != 'LXC')) { + && (jsonObj.hypervisor != 'LXC')) { allowedActions.push("snapshot"); } From 1e8476d38a44037f7d5c24d7d2bb7e9c131a19a0 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Tue, 6 Jan 2015 10:23:44 +0100 Subject: [PATCH 028/391] CLOUDSTACK-8140: CS fails to start after secstorage/consoleproxy.service.offering is set to uuid (cherry picked from commit 84c44b63140e8acf2336549040bda9c0a8f2ff66) --- .../src/com/cloud/configuration/Config.java | 6 +++--- .../consoleproxy/ConsoleProxyManagerImpl.java | 18 ++++++++--------- .../SecondaryStorageManagerImpl.java | 20 ++++++++++++------- setup/db/db/schema-442to450.sql | 2 ++ 4 files changed, 26 insertions(+), 20 deletions(-) diff --git a/server/src/com/cloud/configuration/Config.java b/server/src/com/cloud/configuration/Config.java index a486d3db5ef..091a3e9740c 100755 --- a/server/src/com/cloud/configuration/Config.java +++ b/server/src/com/cloud/configuration/Config.java @@ -1783,7 +1783,7 @@ public enum Config { ConsoleProxyServiceOffering( "Advanced", ManagementServer.class, - Long.class, + String.class, "consoleproxy.service.offering", null, "Uuid of the service offering used by console proxy; if NULL - system offering will be used", @@ -1791,10 +1791,10 @@ public enum Config { SecondaryStorageServiceOffering( "Advanced", ManagementServer.class, - Long.class, + String.class, "secstorage.service.offering", null, - "Service offering used by secondary storage; if NULL - system offering will be used", + "Uuid of the service offering used by secondary storage; if NULL - system offering will be used", null), HaTag("Advanced", ManagementServer.class, String.class, "ha.tag", null, "HA tag defining that the host marked with this tag can be used for HA purposes only", null), ImplicitHostTags( diff --git a/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java b/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java index 14d782a47ef..7f2b52ab4cb 100755 --- a/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java +++ b/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java @@ -91,7 +91,6 @@ import com.cloud.network.dao.IPAddressVO; import com.cloud.network.dao.NetworkDao; import com.cloud.network.dao.NetworkVO; import com.cloud.network.rules.RulesManager; -import com.cloud.offering.DiskOffering; import com.cloud.offering.NetworkOffering; import com.cloud.offering.ServiceOffering; import com.cloud.offerings.dao.NetworkOfferingDao; @@ -105,7 +104,6 @@ import com.cloud.storage.Storage; import com.cloud.storage.StoragePoolStatus; import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; import com.cloud.storage.VMTemplateVO; -import com.cloud.storage.dao.DiskOfferingDao; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.user.Account; import com.cloud.user.AccountManager; @@ -196,8 +194,6 @@ public class ConsoleProxyManagerImpl extends ManagerBase implements ConsoleProxy @Inject private ServiceOfferingDao _offeringDao; @Inject - private DiskOfferingDao _diskOfferingDao; - @Inject private NetworkOfferingDao _networkOfferingDao; @Inject private PrimaryDataStoreDao _storagePoolDao; @@ -1254,13 +1250,15 @@ public class ConsoleProxyManagerImpl extends ManagerBase implements ConsoleProxy //check if there is a default service offering configured String cpvmSrvcOffIdStr = configs.get(Config.ConsoleProxyServiceOffering.key()); if (cpvmSrvcOffIdStr != null) { - DiskOffering diskOffering = _diskOfferingDao.findByUuid(cpvmSrvcOffIdStr); - if (diskOffering == null) { - diskOffering = _diskOfferingDao.findById(Long.parseLong(cpvmSrvcOffIdStr)); + _serviceOffering = _offeringDao.findByUuid(cpvmSrvcOffIdStr); + if (_serviceOffering == null) { + try { + _serviceOffering = _offeringDao.findById(Long.parseLong(cpvmSrvcOffIdStr)); + } catch (NumberFormatException ex) { + s_logger.debug("The system service offering specified by global config is not id, but uuid=" + cpvmSrvcOffIdStr + " for console proxy vm"); + } } - if (diskOffering != null) { - _serviceOffering = _offeringDao.findById(diskOffering.getId()); - } else { + if (_serviceOffering == null) { s_logger.warn("Can't find system service offering specified by global config, uuid=" + cpvmSrvcOffIdStr + " for console proxy vm"); } } diff --git a/services/secondary-storage/controller/src/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java b/services/secondary-storage/controller/src/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java index 86d94df9e8a..25e5b85b599 100755 --- a/services/secondary-storage/controller/src/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java +++ b/services/secondary-storage/controller/src/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java @@ -842,14 +842,20 @@ public class SecondaryStorageManagerImpl extends ManagerBase implements Secondar //check if there is a default service offering configured String ssvmSrvcOffIdStr = configs.get(Config.SecondaryStorageServiceOffering.key()); if (ssvmSrvcOffIdStr != null) { - Long ssvmSrvcOffId = Long.parseLong(ssvmSrvcOffIdStr); - _serviceOffering = _offeringDao.findById(ssvmSrvcOffId); - if (_serviceOffering == null || !_serviceOffering.getSystemUse()) { - String msg = "Can't find system service offering id=" + ssvmSrvcOffId + " for secondary storage vm"; - s_logger.error(msg); - throw new ConfigurationException(msg); + _serviceOffering = _offeringDao.findByUuid(ssvmSrvcOffIdStr); + if (_serviceOffering == null) { + try { + _serviceOffering = _offeringDao.findById(Long.parseLong(ssvmSrvcOffIdStr)); + } catch (NumberFormatException ex) { + s_logger.debug("The system service offering specified by global config is not id, but uuid=" + ssvmSrvcOffIdStr + " for secondary storage vm"); + } } - } else { + if (_serviceOffering == null) { + s_logger.warn("Can't find system service offering specified by global config, uuid=" + ssvmSrvcOffIdStr + " for secondary storage vm"); + } + } + + if(_serviceOffering == null || !_serviceOffering.getSystemUse()){ int ramSize = NumbersUtil.parseInt(_configDao.getValue("ssvm.ram.size"), DEFAULT_SS_VM_RAMSIZE); int cpuFreq = NumbersUtil.parseInt(_configDao.getValue("ssvm.cpu.mhz"), DEFAULT_SS_VM_CPUMHZ); _useLocalStorage = Boolean.parseBoolean(configs.get(Config.SystemVMUseLocalStorage.key())); diff --git a/setup/db/db/schema-442to450.sql b/setup/db/db/schema-442to450.sql index 60c3e45f957..b268eca1df4 100644 --- a/setup/db/db/schema-442to450.sql +++ b/setup/db/db/schema-442to450.sql @@ -990,3 +990,5 @@ INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervis INSERT IGNORE INTO `cloud`.`configuration` (`category`, `instance`, `component`, `name`, `value`, `default_value`, `description`) VALUES ('Advanced', 'DEFAULT', 'ManagementServer', 'xen.heartbeat.timeout' , '180', '120', 'Timeout value to send to the xenheartbeat script for guarding the self fencing functionality'); +UPDATE `cloud`.`configuration` SET description='Uuid of the service offering used by secondary storage; if NULL - system offering will be used' where name='secstorage.service.offering'; + From b2b496288d164fead2c089fb48319e1988b03ce8 Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Mon, 12 Jan 2015 13:33:57 +0530 Subject: [PATCH 029/391] CLOUDSTACK-8034: Hash user IDs for SAML authentication The User table's UUID column is restricted to 40 chars only, since we don't know how long the nameID/userID of a SAML authenticated user will be - the fix hashes that user ID and takes a substring of length 40 chars. For hashing, SHA256 is used which returns a 64 char length string. - Fix tests, add test cases - Improve checkSAMLUser method - Use SHA256 one way hashing to create unique UUID for SAML users Signed-off-by: Rohit Yadav --- .../saml/SAML2UserAuthenticator.java | 2 +- .../SAML2UserAuthenticatorTest.java | 28 ++++++++++++++----- .../cloudstack/utils/auth/SAMLUtils.java | 18 ++++++++---- .../cloudstack/utils/auth/SAMLUtilsTest.java | 10 +++++-- 4 files changed, 43 insertions(+), 15 deletions(-) diff --git a/plugins/user-authenticators/saml2/src/org/apache/cloudstack/saml/SAML2UserAuthenticator.java b/plugins/user-authenticators/saml2/src/org/apache/cloudstack/saml/SAML2UserAuthenticator.java index e623fc21798..31a93a43780 100644 --- a/plugins/user-authenticators/saml2/src/org/apache/cloudstack/saml/SAML2UserAuthenticator.java +++ b/plugins/user-authenticators/saml2/src/org/apache/cloudstack/saml/SAML2UserAuthenticator.java @@ -48,7 +48,7 @@ public class SAML2UserAuthenticator extends DefaultUserAuthenticator { return new Pair(false, null); } else { User user = _userDao.getUser(userAccount.getId()); - if (user != null && SAMLUtils.checkSAMLUserId(user.getUuid()) && + if (user != null && SAMLUtils.checkSAMLUser(user.getUuid(), username) && requestParameters != null && requestParameters.containsKey(SAMLUtils.SAML_RESPONSE)) { return new Pair(true, null); } diff --git a/plugins/user-authenticators/saml2/test/org/apache/cloudstack/SAML2UserAuthenticatorTest.java b/plugins/user-authenticators/saml2/test/org/apache/cloudstack/SAML2UserAuthenticatorTest.java index 29fb4969e83..83792c64d0f 100644 --- a/plugins/user-authenticators/saml2/test/org/apache/cloudstack/SAML2UserAuthenticatorTest.java +++ b/plugins/user-authenticators/saml2/test/org/apache/cloudstack/SAML2UserAuthenticatorTest.java @@ -73,14 +73,28 @@ public class SAML2UserAuthenticatorTest { Mockito.when(userAccountDao.getUserAccount(Mockito.anyString(), Mockito.anyLong())).thenReturn(account); Mockito.when(userDao.getUser(Mockito.anyLong())).thenReturn(user); - // When there is no SAMLRequest in params - Pair pair1 = authenticator.authenticate(SAMLUtils.createSAMLId("user1234"), "random", 1l, null); - Assert.assertFalse(pair1.first()); - - // When there is SAMLRequest in params + Pair pair; Map params = new HashMap(); + + // When there is no SAMLRequest in params + pair = authenticator.authenticate("someUID", "random", 1l, params); + Assert.assertFalse(pair.first()); + + // When there is SAMLRequest in params and user is same as the mocked one params.put(SAMLUtils.SAML_RESPONSE, new Object[]{}); - Pair pair2 = authenticator.authenticate(SAMLUtils.createSAMLId("user1234"), "random", 1l, params); - Assert.assertTrue(pair2.first()); + pair = authenticator.authenticate("someUID", "random", 1l, params); + Assert.assertTrue(pair.first()); + + // When there is SAMLRequest in params but username is null + pair = authenticator.authenticate(null, "random", 1l, params); + Assert.assertFalse(pair.first()); + + // When there is SAMLRequest in params but username is empty + pair = authenticator.authenticate("", "random", 1l, params); + Assert.assertFalse(pair.first()); + + // When there is SAMLRequest in params but username is not valid + pair = authenticator.authenticate("someOtherUID", "random", 1l, params); + Assert.assertFalse(pair.first()); } } diff --git a/utils/src/org/apache/cloudstack/utils/auth/SAMLUtils.java b/utils/src/org/apache/cloudstack/utils/auth/SAMLUtils.java index d129309ba98..dbd2d6fac52 100644 --- a/utils/src/org/apache/cloudstack/utils/auth/SAMLUtils.java +++ b/utils/src/org/apache/cloudstack/utils/auth/SAMLUtils.java @@ -20,6 +20,7 @@ package org.apache.cloudstack.utils.auth; import com.cloud.utils.HttpUtils; +import org.apache.commons.codec.digest.DigestUtils; import org.apache.log4j.Logger; import org.bouncycastle.jce.provider.BouncyCastleProvider; import org.bouncycastle.x509.X509V1CertificateGenerator; @@ -96,18 +97,25 @@ public class SAMLUtils { public static final Logger s_logger = Logger.getLogger(SAMLUtils.class); public static final String SAML_RESPONSE = "SAMLResponse"; - public static final String SAML_NS = "saml://"; + public static final String SAML_NS = "SAML-"; public static final String SAML_NAMEID = "SAML_NAMEID"; public static final String SAML_SESSION = "SAML_SESSION"; public static final String CERTIFICATE_NAME = "SAMLSP_CERTIFICATE"; public static String createSAMLId(String uid) { - String samlUuid = SAML_NS + uid; - return samlUuid.length() > 40 ? samlUuid.substring(0, 40) : samlUuid; + if (uid == null) { + return null; + } + String hash = DigestUtils.sha256Hex(uid); + String samlUuid = SAML_NS + hash; + return samlUuid.substring(0, 40); } - public static Boolean checkSAMLUserId(String uuid) { - return uuid.startsWith(SAML_NS); + public static boolean checkSAMLUser(String uuid, String username) { + if (uuid == null || uuid.isEmpty() || username == null || username.isEmpty()) { + return false; + } + return uuid.startsWith(SAML_NS) && createSAMLId(username).equals(uuid); } public static String generateSecureRandomId() { diff --git a/utils/test/org/apache/cloudstack/utils/auth/SAMLUtilsTest.java b/utils/test/org/apache/cloudstack/utils/auth/SAMLUtilsTest.java index 85be2ef9ad0..bebfd130441 100644 --- a/utils/test/org/apache/cloudstack/utils/auth/SAMLUtilsTest.java +++ b/utils/test/org/apache/cloudstack/utils/auth/SAMLUtilsTest.java @@ -34,8 +34,14 @@ public class SAMLUtilsTest extends TestCase { @Test public void testSAMLId() throws Exception { - assertTrue(SAMLUtils.checkSAMLUserId(SAMLUtils.createSAMLId("someUID"))); - assertFalse(SAMLUtils.checkSAMLUserId("randomUID")); + assertEquals(SAMLUtils.createSAMLId(null), null); + assertEquals(SAMLUtils.createSAMLId("someUserName"), "SAML-305e19dd2581f33fd90b3949298ec8b17de"); + + assertTrue(SAMLUtils.checkSAMLUser(SAMLUtils.createSAMLId("someUserName"), "someUserName")); + assertFalse(SAMLUtils.checkSAMLUser(SAMLUtils.createSAMLId("someUserName"), "someOtherUserName")); + assertFalse(SAMLUtils.checkSAMLUser(SAMLUtils.createSAMLId(null), "someOtherUserName")); + assertFalse(SAMLUtils.checkSAMLUser("randomUID", "randomUID")); + assertFalse(SAMLUtils.checkSAMLUser(null, null)); } @Test From 734bd70173c36508f0fc13a30c3aa8006814c019 Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Mon, 12 Jan 2015 13:56:25 +0530 Subject: [PATCH 030/391] CLOUDSTACK-8037: URL encode cookie values with UTF8 as per version 1 As per Version 1 cookies, certain characters are now allowed such as space, colons etc but they should be url encoded using UTF8 encoding. The frontend has a cookie value unboxing method that removes any double quotes that are added. As per the doc http://download.oracle.com/javase/6/docs/api/java/net/URLEncoder.html values are application/x-www-form-urlencoded and as per http://www.w3.org/TR/html4/interact/forms.html#h-17.13.4 whitespaces are encoded as +, therefore '+' are replaced by %20 (whitespace). Signed-off-by: Rohit Yadav --- .../api/command/SAML2LoginAPIAuthenticatorCmd.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/user-authenticators/saml2/src/org/apache/cloudstack/api/command/SAML2LoginAPIAuthenticatorCmd.java b/plugins/user-authenticators/saml2/src/org/apache/cloudstack/api/command/SAML2LoginAPIAuthenticatorCmd.java index c838ece15cc..2ac1ff54911 100644 --- a/plugins/user-authenticators/saml2/src/org/apache/cloudstack/api/command/SAML2LoginAPIAuthenticatorCmd.java +++ b/plugins/user-authenticators/saml2/src/org/apache/cloudstack/api/command/SAML2LoginAPIAuthenticatorCmd.java @@ -269,14 +269,14 @@ public class SAML2LoginAPIAuthenticatorCmd extends BaseCmd implements APIAuthent try { if (_apiServer.verifyUser(user.getId())) { LoginCmdResponse loginResponse = (LoginCmdResponse) _apiServer.loginUser(session, username, user.getPassword(), domainId, null, remoteAddress, params); - resp.addCookie(new Cookie("userid", loginResponse.getUserId())); - resp.addCookie(new Cookie("domainid", loginResponse.getDomainId())); - resp.addCookie(new Cookie("role", loginResponse.getType())); + resp.addCookie(new Cookie("userid", URLEncoder.encode(loginResponse.getUserId(), HttpUtils.UTF_8))); + resp.addCookie(new Cookie("domainid", URLEncoder.encode(loginResponse.getDomainId(), HttpUtils.UTF_8))); + resp.addCookie(new Cookie("role", URLEncoder.encode(loginResponse.getType(), HttpUtils.UTF_8))); resp.addCookie(new Cookie("username", URLEncoder.encode(loginResponse.getUsername(), HttpUtils.UTF_8))); resp.addCookie(new Cookie("sessionKey", URLEncoder.encode(loginResponse.getSessionKey(), HttpUtils.UTF_8))); resp.addCookie(new Cookie("account", URLEncoder.encode(loginResponse.getAccount(), HttpUtils.UTF_8))); resp.addCookie(new Cookie("timezone", URLEncoder.encode(loginResponse.getTimeZone(), HttpUtils.UTF_8))); - resp.addCookie(new Cookie("userfullname", loginResponse.getFirstName() + "%20" + loginResponse.getLastName())); + resp.addCookie(new Cookie("userfullname", URLEncoder.encode(loginResponse.getFirstName() + " " + loginResponse.getLastName(), HttpUtils.UTF_8).replace("+", "%20"))); resp.sendRedirect(_configDao.getValue(Config.SAMLCloudStackRedirectionUrl.key())); return ApiResponseSerializer.toSerializedString(loginResponse, responseType); From 43587143811b222ca131b0e1237f9e99cd94694d Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Mon, 12 Jan 2015 16:44:23 +0530 Subject: [PATCH 031/391] CLOUDSTACK-8035: Generate and store X509Cert and reuse this for SAML The fix generates X509Certificate if missing from DB and uses that for eternity. SAML SP metadata remains same since it's using the same X509 certificate and it remains same after restarts. The certificate is serialized, base64 encoded and stored in the keystore table under a specific name. For reading, it's retrieved, base64 decoded and deserialized. Signed-off-by: Rohit Yadav --- .../cloudstack/saml/SAML2AuthManagerImpl.java | 39 +++++++++++++++---- .../cloudstack/utils/auth/SAMLUtils.java | 3 +- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/plugins/user-authenticators/saml2/src/org/apache/cloudstack/saml/SAML2AuthManagerImpl.java b/plugins/user-authenticators/saml2/src/org/apache/cloudstack/saml/SAML2AuthManagerImpl.java index 3178f31edb3..f175081f60a 100644 --- a/plugins/user-authenticators/saml2/src/org/apache/cloudstack/saml/SAML2AuthManagerImpl.java +++ b/plugins/user-authenticators/saml2/src/org/apache/cloudstack/saml/SAML2AuthManagerImpl.java @@ -27,6 +27,7 @@ import org.apache.cloudstack.framework.security.keystore.KeystoreDao; import org.apache.cloudstack.framework.security.keystore.KeystoreVO; import org.apache.cloudstack.utils.auth.SAMLUtils; import org.apache.log4j.Logger; +import org.apache.commons.codec.binary.Base64; import org.opensaml.DefaultBootstrap; import org.opensaml.common.xml.SAMLConstants; import org.opensaml.saml2.metadata.EntityDescriptor; @@ -45,6 +46,12 @@ import org.springframework.stereotype.Component; import javax.ejb.Local; import javax.inject.Inject; import javax.xml.stream.FactoryConfigurationError; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutput; +import java.io.ObjectOutputStream; import java.security.InvalidKeyException; import java.security.KeyPair; import java.security.NoSuchAlgorithmException; @@ -94,12 +101,12 @@ public class SAML2AuthManagerImpl extends AdapterBase implements SAML2AuthManage } private boolean setup() { - KeystoreVO keyStoreVO = _ksDao.findByName(SAMLUtils.CERTIFICATE_NAME); + KeystoreVO keyStoreVO = _ksDao.findByName(SAMLUtils.SAMLSP_KEYPAIR); if (keyStoreVO == null) { try { KeyPair keyPair = SAMLUtils.generateRandomKeyPair(); - _ksDao.save(SAMLUtils.CERTIFICATE_NAME, SAMLUtils.savePrivateKey(keyPair.getPrivate()), SAMLUtils.savePublicKey(keyPair.getPublic()), "saml-sp"); - keyStoreVO = _ksDao.findByName(SAMLUtils.CERTIFICATE_NAME); + _ksDao.save(SAMLUtils.SAMLSP_KEYPAIR, SAMLUtils.savePrivateKey(keyPair.getPrivate()), SAMLUtils.savePublicKey(keyPair.getPublic()), "samlsp-keypair"); + keyStoreVO = _ksDao.findByName(SAMLUtils.SAMLSP_KEYPAIR); } catch (NoSuchProviderException | NoSuchAlgorithmException e) { s_logger.error("Unable to create and save SAML keypair"); } @@ -110,10 +117,28 @@ public class SAML2AuthManagerImpl extends AdapterBase implements SAML2AuthManage PublicKey publicKey = SAMLUtils.loadPublicKey(keyStoreVO.getKey()); if (privateKey != null && publicKey != null) { spKeyPair = new KeyPair(publicKey, privateKey); - try { - spX509Key = SAMLUtils.generateRandomX509Certificate(spKeyPair); - } catch (NoSuchAlgorithmException | NoSuchProviderException | CertificateEncodingException | SignatureException | InvalidKeyException e) { - s_logger.error("SAML Plugin won't be able to use X509 signed authentication"); + KeystoreVO x509VO = _ksDao.findByName(SAMLUtils.SAMLSP_X509CERT); + if (x509VO == null) { + try { + spX509Key = SAMLUtils.generateRandomX509Certificate(spKeyPair); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + ObjectOutput out = new ObjectOutputStream(bos); + out.writeObject(spX509Key); + out.flush(); + _ksDao.save(SAMLUtils.SAMLSP_X509CERT, Base64.encodeBase64String(bos.toByteArray()), "", "samlsp-x509cert"); + bos.close(); + } catch (NoSuchAlgorithmException | NoSuchProviderException | CertificateEncodingException | SignatureException | InvalidKeyException | IOException e) { + s_logger.error("SAML Plugin won't be able to use X509 signed authentication"); + } + } else { + try { + ByteArrayInputStream bi = new ByteArrayInputStream(Base64.decodeBase64(x509VO.getCertificate())); + ObjectInputStream si = new ObjectInputStream(bi); + spX509Key = (X509Certificate) si.readObject(); + bi.close(); + } catch (IOException | ClassNotFoundException ignored) { + s_logger.error("SAML Plugin won't be able to use X509 signed authentication. Failed to load X509 Certificate from Database."); + } } } } diff --git a/utils/src/org/apache/cloudstack/utils/auth/SAMLUtils.java b/utils/src/org/apache/cloudstack/utils/auth/SAMLUtils.java index dbd2d6fac52..bb4af3a57d5 100644 --- a/utils/src/org/apache/cloudstack/utils/auth/SAMLUtils.java +++ b/utils/src/org/apache/cloudstack/utils/auth/SAMLUtils.java @@ -100,7 +100,8 @@ public class SAMLUtils { public static final String SAML_NS = "SAML-"; public static final String SAML_NAMEID = "SAML_NAMEID"; public static final String SAML_SESSION = "SAML_SESSION"; - public static final String CERTIFICATE_NAME = "SAMLSP_CERTIFICATE"; + public static final String SAMLSP_KEYPAIR = "SAMLSP_KEYPAIR"; + public static final String SAMLSP_X509CERT = "SAMLSP_X509CERT"; public static String createSAMLId(String uid) { if (uid == null) { From 2720a1b49fb6bcd4771a043fabcbac945e7b389e Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Mon, 12 Jan 2015 12:25:28 +0100 Subject: [PATCH 032/391] CLOUDSTACK-8146: Resource count of primary storage does not consider the detached volumes (cherry picked from commit b528047fb6f1c199e5e1cfe991a10b72a2b32a49) --- .../com/cloud/storage/dao/VolumeDaoImpl.java | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/engine/schema/src/com/cloud/storage/dao/VolumeDaoImpl.java b/engine/schema/src/com/cloud/storage/dao/VolumeDaoImpl.java index 24de7172e06..61cce8dc70f 100755 --- a/engine/schema/src/com/cloud/storage/dao/VolumeDaoImpl.java +++ b/engine/schema/src/com/cloud/storage/dao/VolumeDaoImpl.java @@ -65,6 +65,7 @@ public class VolumeDaoImpl extends GenericDaoBase implements Vol protected final SearchBuilder AllFieldsSearch; protected GenericSearchBuilder CountByAccount; protected GenericSearchBuilder primaryStorageSearch; + protected GenericSearchBuilder primaryStorageSearch2; protected GenericSearchBuilder secondaryStorageSearch; @Inject ResourceTagDao _tagsDao; @@ -367,7 +368,6 @@ public class VolumeDaoImpl extends GenericDaoBase implements Vol primaryStorageSearch = createSearchBuilder(SumCount.class); primaryStorageSearch.select("sum", Func.SUM, primaryStorageSearch.entity().getSize()); primaryStorageSearch.and("accountId", primaryStorageSearch.entity().getAccountId(), Op.EQ); - primaryStorageSearch.and("virtualRouterVmIds", primaryStorageSearch.entity().getInstanceId(), Op.NIN); primaryStorageSearch.and().op("path", primaryStorageSearch.entity().getPath(), Op.NNULL); primaryStorageSearch.or("states", primaryStorageSearch.entity().getState(), Op.IN); primaryStorageSearch.cp(); @@ -375,6 +375,18 @@ public class VolumeDaoImpl extends GenericDaoBase implements Vol primaryStorageSearch.and("isRemoved", primaryStorageSearch.entity().getRemoved(), Op.NULL); primaryStorageSearch.done(); + primaryStorageSearch2 = createSearchBuilder(SumCount.class); + primaryStorageSearch2.select("sum", Func.SUM, primaryStorageSearch2.entity().getSize()); + primaryStorageSearch2.and("accountId", primaryStorageSearch2.entity().getAccountId(), Op.EQ); + primaryStorageSearch2.and().op("instanceId", primaryStorageSearch2.entity().getInstanceId(), Op.NULL); + primaryStorageSearch2.or("virtualRouterVmIds", primaryStorageSearch2.entity().getInstanceId(), Op.NIN); + primaryStorageSearch2.cp(); + primaryStorageSearch2.and().op("path", primaryStorageSearch2.entity().getPath(), Op.NNULL); + primaryStorageSearch2.or("states", primaryStorageSearch2.entity().getState(), Op.IN); + primaryStorageSearch2.cp(); + primaryStorageSearch2.and("displayVolume", primaryStorageSearch2.entity().isDisplayVolume(), Op.EQ); + primaryStorageSearch2.and("isRemoved", primaryStorageSearch2.entity().getRemoved(), Op.NULL); + secondaryStorageSearch = createSearchBuilder(SumCount.class); secondaryStorageSearch.select("sum", Func.SUM, secondaryStorageSearch.entity().getSize()); secondaryStorageSearch.and("accountId", secondaryStorageSearch.entity().getAccountId(), Op.EQ); @@ -405,11 +417,14 @@ public class VolumeDaoImpl extends GenericDaoBase implements Vol @Override public long primaryStorageUsedForAccount(long accountId, List virtualRouters) { - SearchCriteria sc = primaryStorageSearch.create(); - sc.setParameters("accountId", accountId); + SearchCriteria sc; if (!virtualRouters.isEmpty()) { + sc = primaryStorageSearch2.create(); sc.setParameters("virtualRouterVmIds", virtualRouters.toArray(new Object[virtualRouters.size()])); + } else { + sc = primaryStorageSearch.create(); } + sc.setParameters("accountId", accountId); sc.setParameters("states", State.Allocated); sc.setParameters("displayVolume", 1); List storageSpace = customSearch(sc, null); From 23de431f96e1dad8a21055ac98926c428e83c775 Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Mon, 12 Jan 2015 18:55:52 +0530 Subject: [PATCH 033/391] CLOUDSTACK-8037: Fix attribute detection, tested to work with onelogin.com Signed-off-by: Rohit Yadav --- .../SAML2LoginAPIAuthenticatorCmd.java | 37 +++++++++++-------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/plugins/user-authenticators/saml2/src/org/apache/cloudstack/api/command/SAML2LoginAPIAuthenticatorCmd.java b/plugins/user-authenticators/saml2/src/org/apache/cloudstack/api/command/SAML2LoginAPIAuthenticatorCmd.java index 2ac1ff54911..a0dcad5c9cb 100644 --- a/plugins/user-authenticators/saml2/src/org/apache/cloudstack/api/command/SAML2LoginAPIAuthenticatorCmd.java +++ b/plugins/user-authenticators/saml2/src/org/apache/cloudstack/api/command/SAML2LoginAPIAuthenticatorCmd.java @@ -239,22 +239,27 @@ public class SAML2LoginAPIAuthenticatorCmd extends BaseCmd implements APIAuthent } } - AttributeStatement attributeStatement = assertion.getAttributeStatements().get(0); - List attributes = attributeStatement.getAttributes(); - - // Try capturing standard LDAP attributes - for (Attribute attribute: attributes) { - String attributeName = attribute.getName(); - String attributeValue = attribute.getAttributeValues().get(0).getDOM().getTextContent(); - if (attributeName.equalsIgnoreCase("uid") && uniqueUserId == null) { - username = attributeValue; - uniqueUserId = SAMLUtils.createSAMLId(username); - } else if (attributeName.equalsIgnoreCase("givenName")) { - firstName = attributeValue; - } else if (attributeName.equalsIgnoreCase(("sn"))) { - lastName = attributeValue; - } else if (attributeName.equalsIgnoreCase("mail")) { - email = attributeValue; + List attributeStatements = assertion.getAttributeStatements(); + if (attributeStatements != null && attributeStatements.size() > 0) { + for (AttributeStatement attributeStatement: attributeStatements) { + if (attributeStatement == null) { + continue; + } + // Try capturing standard LDAP attributes + for (Attribute attribute: attributeStatement.getAttributes()) { + String attributeName = attribute.getName(); + String attributeValue = attribute.getAttributeValues().get(0).getDOM().getTextContent(); + if (attributeName.equalsIgnoreCase("uid") && uniqueUserId == null) { + username = attributeValue; + uniqueUserId = SAMLUtils.createSAMLId(username); + } else if (attributeName.equalsIgnoreCase("givenName")) { + firstName = attributeValue; + } else if (attributeName.equalsIgnoreCase(("sn"))) { + lastName = attributeValue; + } else if (attributeName.equalsIgnoreCase("mail")) { + email = attributeValue; + } + } } } From 94e6b7587d83b4ac1a0af0953199e0f4bd9340e8 Mon Sep 17 00:00:00 2001 From: Daan Hoogland Date: Mon, 12 Jan 2015 20:40:01 +0100 Subject: [PATCH 034/391] CID-1256275 regression: resource leak in systemvm update code (cherry picked from commit 06d4458d0a9de5be7a7bf590678eb4b03989e9a1) Conflicts: engine/schema/src/com/cloud/upgrade/dao/Upgrade442to450.java --- .../cloud/upgrade/dao/Upgrade442to450.java | 252 +++++++++--------- 1 file changed, 127 insertions(+), 125 deletions(-) diff --git a/engine/schema/src/com/cloud/upgrade/dao/Upgrade442to450.java b/engine/schema/src/com/cloud/upgrade/dao/Upgrade442to450.java index e8a517b236b..191e022d623 100644 --- a/engine/schema/src/com/cloud/upgrade/dao/Upgrade442to450.java +++ b/engine/schema/src/com/cloud/upgrade/dao/Upgrade442to450.java @@ -140,9 +140,6 @@ public class Upgrade442to450 implements DbUpgrade { } private void upgradeMemoryOfInternalLoadBalancervmOffering(Connection conn) { - PreparedStatement updatePstmt = null; - PreparedStatement selectPstmt = null; - ResultSet selectResultSet = null; int newRamSize = 256; //256MB long serviceOfferingId = 0; @@ -151,10 +148,9 @@ public class Upgrade442to450 implements DbUpgrade { * We should not update/modify any user-defined offering. */ - try { - selectPstmt = conn.prepareStatement("SELECT id FROM `cloud`.`service_offering` WHERE vm_type='internalloadbalancervm'"); - updatePstmt = conn.prepareStatement("UPDATE `cloud`.`service_offering` SET ram_size=? WHERE id=?"); - selectResultSet = selectPstmt.executeQuery(); + try (PreparedStatement selectPstmt = conn.prepareStatement("SELECT id FROM `cloud`.`service_offering` WHERE vm_type='internalloadbalancervm'"); + PreparedStatement updatePstmt = conn.prepareStatement("UPDATE `cloud`.`service_offering` SET ram_size=? WHERE id=?"); + ResultSet selectResultSet = selectPstmt.executeQuery()){ if(selectResultSet.next()) { serviceOfferingId = selectResultSet.getLong("id"); } @@ -164,19 +160,6 @@ public class Upgrade442to450 implements DbUpgrade { updatePstmt.executeUpdate(); } catch (SQLException e) { throw new CloudRuntimeException("Unable to upgrade ram_size of service offering for internal loadbalancer vm. ", e); - } finally { - try { - if (selectPstmt != null) { - selectPstmt.close(); - } - if (selectResultSet != null) { - selectResultSet.close(); - } - if (updatePstmt != null) { - updatePstmt.close(); - } - } catch (SQLException e) { - } } s_logger.debug("Done upgrading RAM for service offering of internal loadbalancer vm to " + newRamSize); } @@ -192,127 +175,146 @@ public class Upgrade442to450 implements DbUpgrade { } private void updateSystemVmTemplates(Connection conn) { - PreparedStatement pstmt = null; - ResultSet rs = null; s_logger.debug("Updating System Vm template IDs"); - try{ - //Get all hypervisors in use - Set hypervisorsListInUse = new HashSet(); - try { - pstmt = conn.prepareStatement("select distinct(hypervisor_type) from `cloud`.`cluster` where removed is null"); - rs = pstmt.executeQuery(); - while(rs.next()){ - switch (Hypervisor.HypervisorType.getType(rs.getString(1))) { - case XenServer: hypervisorsListInUse.add(Hypervisor.HypervisorType.XenServer); - break; - case KVM: hypervisorsListInUse.add(Hypervisor.HypervisorType.KVM); - break; - case VMware: hypervisorsListInUse.add(Hypervisor.HypervisorType.VMware); - break; - case Hyperv: hypervisorsListInUse.add(Hypervisor.HypervisorType.Hyperv); - break; - case LXC: hypervisorsListInUse.add(Hypervisor.HypervisorType.LXC); - break; - } + //Get all hypervisors in use + Set hypervisorsListInUse = new HashSet(); + try (PreparedStatement pstmt = conn.prepareStatement("select distinct(hypervisor_type) from `cloud`.`cluster` where removed is null"); + ResultSet rs = pstmt.executeQuery() + ) { + while(rs.next()){ + switch (Hypervisor.HypervisorType.getType(rs.getString(1))) { + case XenServer: hypervisorsListInUse.add(Hypervisor.HypervisorType.XenServer); + break; + case KVM: hypervisorsListInUse.add(Hypervisor.HypervisorType.KVM); + break; + case VMware: hypervisorsListInUse.add(Hypervisor.HypervisorType.VMware); + break; + case Hyperv: hypervisorsListInUse.add(Hypervisor.HypervisorType.Hyperv); + break; + case LXC: hypervisorsListInUse.add(Hypervisor.HypervisorType.LXC); + break; + default: // no action on cases Any, BareMetal, None, Ovm, Parralels, Simulator and VirtualBox: + break; } - } catch (SQLException e) { - throw new CloudRuntimeException("Error while listing hypervisors in use", e); } + } catch (SQLException e) { + s_logger.error("updateSystemVmTemplates:Exception while getting hypervisor types from clusters: "+e.getMessage()); + throw new CloudRuntimeException("updateSystemVmTemplates:Exception while getting hypervisor types from clusters", e); + } - Map NewTemplateNameList = new HashMap(){ - { put(Hypervisor.HypervisorType.XenServer, "systemvm-xenserver-4.5"); - put(Hypervisor.HypervisorType.VMware, "systemvm-vmware-4.5"); - put(Hypervisor.HypervisorType.KVM, "systemvm-kvm-4.5"); - put(Hypervisor.HypervisorType.LXC, "systemvm-lxc-4.5"); - put(Hypervisor.HypervisorType.Hyperv, "systemvm-hyperv-4.5"); - } - }; + Map NewTemplateNameList = new HashMap() { + { + put(Hypervisor.HypervisorType.XenServer, "systemvm-xenserver-4.5"); + put(Hypervisor.HypervisorType.VMware, "systemvm-vmware-4.5"); + put(Hypervisor.HypervisorType.KVM, "systemvm-kvm-4.5"); + put(Hypervisor.HypervisorType.LXC, "systemvm-lxc-4.5"); + put(Hypervisor.HypervisorType.Hyperv, "systemvm-hyperv-4.5"); + } + }; - Map routerTemplateConfigurationNames = new HashMap(){ - { put(Hypervisor.HypervisorType.XenServer, "router.template.xen"); - put(Hypervisor.HypervisorType.VMware, "router.template.vmware"); - put(Hypervisor.HypervisorType.KVM, "router.template.kvm"); - put(Hypervisor.HypervisorType.LXC, "router.template.lxc"); - put(Hypervisor.HypervisorType.Hyperv, "router.template.hyperv"); - } - }; + Map routerTemplateConfigurationNames = new HashMap() { + { + put(Hypervisor.HypervisorType.XenServer, "router.template.xen"); + put(Hypervisor.HypervisorType.VMware, "router.template.vmware"); + put(Hypervisor.HypervisorType.KVM, "router.template.kvm"); + put(Hypervisor.HypervisorType.LXC, "router.template.lxc"); + put(Hypervisor.HypervisorType.Hyperv, "router.template.hyperv"); + } + }; - Map newTemplateUrl = new HashMap(){ - { put(Hypervisor.HypervisorType.XenServer, "http://download.cloud.com/templates/4.5/systemvm64template-4.5-xen.vhd.bz2"); - put(Hypervisor.HypervisorType.VMware, "http://download.cloud.com/templates/4.5/systemvm64template-4.5-vmware.ova"); - put(Hypervisor.HypervisorType.KVM, "http://download.cloud.com/templates/4.5/systemvm64template-4.5-kvm.qcow2.bz2"); - put(Hypervisor.HypervisorType.LXC, "http://download.cloud.com/templates/4.5/systemvm64template-4.5-kvm.qcow2.bz2"); - put(Hypervisor.HypervisorType.Hyperv, "http://download.cloud.com/templates/4.5/systemvm64template-4.5-hyperv.vhd.zip"); - } - }; + Map newTemplateUrl = new HashMap() { + { + put(Hypervisor.HypervisorType.XenServer, "http://download.cloud.com/templates/4.5/systemvm64template-4.5-xen.vhd.bz2"); + put(Hypervisor.HypervisorType.VMware, "http://download.cloud.com/templates/4.5/systemvm64template-4.5-vmware.ova"); + put(Hypervisor.HypervisorType.KVM, "http://download.cloud.com/templates/4.5/systemvm64template-4.5-kvm.qcow2.bz2"); + put(Hypervisor.HypervisorType.LXC, "http://download.cloud.com/templates/4.5/systemvm64template-4.5-kvm.qcow2.bz2"); + put(Hypervisor.HypervisorType.Hyperv, "http://download.cloud.com/templates/4.5/systemvm64template-4.5-hyperv.vhd.zip"); + } + }; - Map newTemplateChecksum = new HashMap(){ - { put(Hypervisor.HypervisorType.XenServer, "2b15ab4401c2d655264732d3fc600241"); - put(Hypervisor.HypervisorType.VMware, "3106a79a4ce66cd7f6a7c50e93f2db57"); - put(Hypervisor.HypervisorType.KVM, "aa9f501fecd3de1daeb9e2f357f6f002"); - put(Hypervisor.HypervisorType.LXC, "aa9f501fecd3de1daeb9e2f357f6f002"); - put(Hypervisor.HypervisorType.Hyperv, "70bd30ea02ee9ed67d2c6b85c179cee9"); - } - }; + Map newTemplateChecksum = new HashMap() { + { + put(Hypervisor.HypervisorType.XenServer, "2b15ab4401c2d655264732d3fc600241"); + put(Hypervisor.HypervisorType.VMware, "3106a79a4ce66cd7f6a7c50e93f2db57"); + put(Hypervisor.HypervisorType.KVM, "aa9f501fecd3de1daeb9e2f357f6f002"); + put(Hypervisor.HypervisorType.LXC, "aa9f501fecd3de1daeb9e2f357f6f002"); + put(Hypervisor.HypervisorType.Hyperv, "70bd30ea02ee9ed67d2c6b85c179cee9"); + } + }; - for (Map.Entry hypervisorAndTemplateName : NewTemplateNameList.entrySet()){ - s_logger.debug("Updating " + hypervisorAndTemplateName.getKey() + " System Vms"); - try { - //Get 4.5.0 system Vm template Id for corresponding hypervisor - pstmt = conn.prepareStatement("select id from `cloud`.`vm_template` where name = ? and removed is null order by id desc limit 1"); - pstmt.setString(1, hypervisorAndTemplateName.getValue()); - rs = pstmt.executeQuery(); + for (Map.Entry hypervisorAndTemplateName : NewTemplateNameList.entrySet()) { + s_logger.debug("Updating " + hypervisorAndTemplateName.getKey() + " System Vms"); + try (PreparedStatement pstmt = conn.prepareStatement("select id from `cloud`.`vm_template` where name = ? and removed is null order by id desc limit 1")) { + //Get 4.5.0 system Vm template Id for corresponding hypervisor + long templateId = -1; + pstmt.setString(1, hypervisorAndTemplateName.getValue()); + try (ResultSet rs = pstmt.executeQuery()) { if(rs.next()){ - long templateId = rs.getLong(1); - rs.close(); - pstmt.close(); - pstmt = conn.prepareStatement("update `cloud`.`vm_template` set type='SYSTEM' where id = ?"); - pstmt.setLong(1, templateId); - pstmt.executeUpdate(); - pstmt.close(); - // update templete ID of system Vms - pstmt = conn.prepareStatement("update `cloud`.`vm_instance` set vm_template_id = ? where type <> 'User' and hypervisor_type = ?"); - pstmt.setLong(1, templateId); - pstmt.setString(2, hypervisorAndTemplateName.getKey().toString()); - pstmt.executeUpdate(); - pstmt.close(); - // Change value of global configuration parameter router.template.* for the corresponding hypervisor - pstmt = conn.prepareStatement("UPDATE `cloud`.`configuration` SET value = ? WHERE name = ?"); - pstmt.setString(1, hypervisorAndTemplateName.getValue()); - pstmt.setString(2, routerTemplateConfigurationNames.get(hypervisorAndTemplateName.getKey())); - pstmt.executeUpdate(); - pstmt.close(); + templateId = rs.getLong(1); + } + } catch (SQLException e) + { + s_logger.error("updateSystemVmTemplates:Exception while getting ids of templates: "+e.getMessage()); + throw new CloudRuntimeException("updateSystemVmTemplates:Exception while getting ids of templates", e); + } + + // change template type to SYSTEM + if (templateId != -1) { + try(PreparedStatement templ_type_pstmt = conn.prepareStatement("update `cloud`.`vm_template` set type='SYSTEM' where id = ?");) + { + templ_type_pstmt.setLong(1, templateId); + templ_type_pstmt.executeUpdate(); + } + catch (SQLException e) + { + s_logger.error("updateSystemVmTemplates:Exception while updating template with id " + templateId + " to be marked as 'system': "+e.getMessage()); + throw new CloudRuntimeException("updateSystemVmTemplates:Exception while updating template with id " + templateId + " to be marked as 'system'", e); + } + // update template ID of system Vms + try(PreparedStatement update_templ_id_pstmt = conn.prepareStatement("update `cloud`.`vm_instance` set vm_template_id = ? where type <> 'User' and hypervisor_type = ?");) + { + update_templ_id_pstmt.setLong(1, templateId); + update_templ_id_pstmt.setString(2, hypervisorAndTemplateName.getKey().toString()); + update_templ_id_pstmt.executeUpdate(); + }catch (Exception e) + { + s_logger.error("updateSystemVmTemplates:Exception while setting template for " + hypervisorAndTemplateName.getKey().toString() + " to " + templateId + ": "+e.getMessage()); + throw new CloudRuntimeException("updateSystemVmTemplates:Exception while setting template for " + hypervisorAndTemplateName.getKey().toString() + " to " + templateId, e); + } + // Change value of global configuration parameter router.template.* for the corresponding hypervisor + try(PreparedStatement update_pstmt = conn.prepareStatement("UPDATE `cloud`.`configuration` SET value = ? WHERE name = ?");) { + update_pstmt.setString(1, hypervisorAndTemplateName.getValue()); + update_pstmt.setString(2, routerTemplateConfigurationNames.get(hypervisorAndTemplateName.getKey())); + update_pstmt.executeUpdate(); + }catch (SQLException e) + { + s_logger.error("updateSystemVmTemplates:Exception while setting " + routerTemplateConfigurationNames.get(hypervisorAndTemplateName.getKey()) + " to " + hypervisorAndTemplateName.getValue() + ": "+e.getMessage()); + throw new CloudRuntimeException("updateSystemVmTemplates:Exception while setting " + routerTemplateConfigurationNames.get(hypervisorAndTemplateName.getKey()) + " to " + hypervisorAndTemplateName.getValue(), e); + } + } else { + if (hypervisorsListInUse.contains(hypervisorAndTemplateName.getKey())){ + throw new CloudRuntimeException("4.5.0 " + hypervisorAndTemplateName.getKey() + " SystemVm template not found. Cannot upgrade system Vms"); } else { - if (hypervisorsListInUse.contains(hypervisorAndTemplateName.getKey())){ - throw new CloudRuntimeException("4.5.0 " + hypervisorAndTemplateName.getKey() + " SystemVm template not found. Cannot upgrade system Vms"); - } else { - s_logger.warn("4.5.0 " + hypervisorAndTemplateName.getKey() + " SystemVm template not found. " + hypervisorAndTemplateName.getKey() + " hypervisor is not used, so not failing upgrade"); - // Update the latest template URLs for corresponding hypervisor - pstmt = conn.prepareStatement("UPDATE `cloud`.`vm_template` SET url = ? , checksum = ? WHERE hypervisor_type = ? AND type = 'SYSTEM' AND removed is null order by id desc limit 1"); - pstmt.setString(1, newTemplateUrl.get(hypervisorAndTemplateName.getKey())); - pstmt.setString(2, newTemplateChecksum.get(hypervisorAndTemplateName.getKey())); - pstmt.setString(3, hypervisorAndTemplateName.getKey().toString()); - pstmt.executeUpdate(); - pstmt.close(); + s_logger.warn("4.5.0 " + hypervisorAndTemplateName.getKey() + " SystemVm template not found. " + hypervisorAndTemplateName.getKey() + " hypervisor is not used, so not failing upgrade"); + // Update the latest template URLs for corresponding hypervisor + try(PreparedStatement update_templ_url_pstmt = conn.prepareStatement("UPDATE `cloud`.`vm_template` SET url = ? , checksum = ? WHERE hypervisor_type = ? AND type = 'SYSTEM' AND removed is null order by id desc limit 1");) { + update_templ_url_pstmt.setString(1, newTemplateUrl.get(hypervisorAndTemplateName.getKey())); + update_templ_url_pstmt.setString(2, newTemplateChecksum.get(hypervisorAndTemplateName.getKey())); + update_templ_url_pstmt.setString(3, hypervisorAndTemplateName.getKey().toString()); + update_templ_url_pstmt.executeUpdate(); + }catch (SQLException e) + { + s_logger.error("updateSystemVmTemplates:Exception while updating 'url' and 'checksum' for hypervisor type " + hypervisorAndTemplateName.getKey().toString() + ": "+e.getMessage()); + throw new CloudRuntimeException("updateSystemVmTemplates:Exception while updating 'url' and 'checksum' for hypervisor type " + hypervisorAndTemplateName.getKey().toString(), e); } } - } catch (SQLException e) { - throw new CloudRuntimeException("Error while updating "+ hypervisorAndTemplateName.getKey() +" systemVm template", e); - } - } - s_logger.debug("Updating System Vm Template IDs Complete"); - } finally { - try { - if (rs != null) { - rs.close(); - } - - if (pstmt != null) { - pstmt.close(); } } catch (SQLException e) { + s_logger.error("updateSystemVmTemplates:Exception while getting ids of templates: "+e.getMessage()); + throw new CloudRuntimeException("updateSystemVmTemplates:Exception while getting ids of templates", e); } } + s_logger.debug("Updating System Vm Template IDs Complete"); } From d161642f6967c92b3c3aeacdda7b920fbe67428f Mon Sep 17 00:00:00 2001 From: David Nalley Date: Mon, 12 Jan 2015 17:46:12 -0500 Subject: [PATCH 035/391] make the date command actually work --- tools/build/build_asf.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 tools/build/build_asf.sh diff --git a/tools/build/build_asf.sh b/tools/build/build_asf.sh old mode 100644 new mode 100755 index aec5b2ec848..45b07aa158d --- a/tools/build/build_asf.sh +++ b/tools/build/build_asf.sh @@ -108,7 +108,7 @@ echo "cloudstack ($version) unstable; urgency=low" >>$tmpfilenm echo >>$tmpfilenm echo " * Update the version to $version" >>$tmpfilenm echo >>$tmpfilenm -echo " -- the Apache CloudStack project `date -j '+%a, %d %b %Y %T %z'`" >>$tmpfilenm +echo " -- the Apache CloudStack project `date '+%a, %d %b %Y %T %z'`" >>$tmpfilenm echo >>$tmpfilenm cat debian/changelog >>$tmpfilenm From 9278ac1130683057cfbd752138b266565bd0f695 Mon Sep 17 00:00:00 2001 From: Gaurav Aradhye Date: Fri, 9 Jan 2015 15:41:03 +0530 Subject: [PATCH 036/391] CLOUDSTACK-8145: Adding new test to test blocker bugs and modifying other test case to work around the bug Signed-off-by: SrikanteswaraRao Talluri --- .../component/test_blocker_bugs.py | 146 +++++++++++++++++- .../component/test_reset_ssh_keypair.py | 6 + 2 files changed, 150 insertions(+), 2 deletions(-) diff --git a/test/integration/component/test_blocker_bugs.py b/test/integration/component/test_blocker_bugs.py index f9530f0177d..9e8b916757b 100644 --- a/test/integration/component/test_blocker_bugs.py +++ b/test/integration/component/test_blocker_bugs.py @@ -106,7 +106,6 @@ class Services: "sleep": 180, } - class TestTemplate(cloudstackTestCase): def setUp(self): @@ -253,7 +252,6 @@ class TestTemplate(cloudstackTestCase): ) return - class TestNATRules(cloudstackTestCase): @classmethod @@ -1001,3 +999,147 @@ class TestTemplates(cloudstackTestCase): "Check the name of the template" ) return + +class TestDataPersistency(cloudstackTestCase): + + @classmethod + def setUpClass(cls): + + cls.testClient = super(TestDataPersistency, cls).getClsTestClient() + cls.api_client = cls.testClient.getApiClient() + + cls.services = cls.testClient.getParsedTestDataConfig() + # Get Zone, Domain and templates + cls.zone = get_zone(cls.api_client, cls.testClient.getZoneForTests()) + cls.domain = get_domain(cls.api_client) + cls.services['mode'] = cls.zone.networktype + template = get_template( + cls.api_client, + cls.zone.id, + cls.services["ostype"] + ) + cls.services["virtual_machine"]["zoneid"] = cls.zone.id + + #Create an account, network, VM and IP addresses + cls.account = Account.create( + cls.api_client, + cls.services["account"], + domainid=cls.domain.id + ) + + cls.userapiclient = cls.testClient.getUserApiClient( + UserName=cls.account.name, + DomainName=cls.account.domain) + + cls.service_offering = ServiceOffering.create( + cls.api_client, + cls.services["service_offering"] + ) + cls.virtual_machine = VirtualMachine.create( + cls.api_client, + cls.services["virtual_machine"], + templateid=template.id, + accountid=cls.account.name, + domainid=cls.account.domainid, + serviceofferingid=cls.service_offering.id, + mode=cls.services["mode"] + ) + cls.cleanup = [ + cls.account, + cls.service_offering + ] + return + + @classmethod + def tearDownClass(cls): + try: + #Clean up, terminate the created templates + cleanup_resources(cls.api_client, cls.cleanup) + + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + return + + def setUp(self): + self.apiclient = self.testClient.getApiClient() + return + + def tearDown(self): + # No need + return + + @attr(tags=["advanced", "basic", "advancedns", "eip"], required_hardware="true") + def test_01_data_persistency_root_disk(self): + """ + Test the timing issue of root disk data sync + + # 1. Write data to root disk of a VM + # 2. Create a template from the root disk of VM + # 3. Create a new VM from this template + # 4. Check that the data is present in the new VM + + This is to test that data is persisted on root disk of VM or not + when template is created immediately from it + """ + + ssh = self.virtual_machine.get_ssh_client() + + sampleText = "This is sample data" + + cmds = [ + "cd /root/", + "touch testFile.txt", + "chmod 600 testFile.txt", + "echo %s >> testFile.txt" % sampleText + ] + for c in cmds: + ssh.execute(c) + + #Stop virtual machine + self.virtual_machine.stop(self.api_client) + + list_volume = Volume.list( + self.api_client, + virtualmachineid=self.virtual_machine.id, + type='ROOT', + listall=True) + + if isinstance(list_volume, list): + self.volume = list_volume[0] + else: + raise Exception( + "Exception: Unable to find root volume for VM: %s" % + self.virtual_machine.id) + + self.services["template"]["ostype"] = self.services["ostype"] + #Create templates for Edit, Delete & update permissions testcases + customTemplate = Template.create( + self.userapiclient, + self.services["template"], + self.volume.id, + account=self.account.name, + domainid=self.account.domainid + ) + self.cleanup.append(customTemplate) + # Delete the VM - No longer needed + self.virtual_machine.delete(self.apiclient) + + virtual_machine = VirtualMachine.create( + self.apiclient, + self.services["virtual_machine"], + templateid=customTemplate.id, + accountid=self.account.name, + domainid=self.account.domainid, + serviceofferingid=self.service_offering.id, + mode=self.services["mode"] + ) + + ssh = virtual_machine.get_ssh_client() + + response = ssh.execute("cat /root/testFile.txt") + res = str(response[0]) + + self.assertEqual(res, sampleText, "The data %s does not match\ + with sample test %s" % + (res, sampleText)) + return diff --git a/test/integration/component/test_reset_ssh_keypair.py b/test/integration/component/test_reset_ssh_keypair.py index a2e743a88b2..db7e9b82d98 100644 --- a/test/integration/component/test_reset_ssh_keypair.py +++ b/test/integration/component/test_reset_ssh_keypair.py @@ -200,6 +200,9 @@ class TestResetSSHKeypair(cloudstackTestCase): for c in cmds: ssh.execute(c) + # Adding delay of 120 sec to avoid data loss due to timing issue + time.sleep(120) + #Stop virtual machine cls.virtual_machine.stop(cls.api_client) @@ -1031,6 +1034,9 @@ class TestResetSSHKeyUserRights(cloudstackTestCase): for c in cmds: ssh.execute(c) + # Adding delay of 120 sec to avoid data loss due to timing issue + time.sleep(120) + try: #Stop virtual machine cls.virtual_machine.stop(cls.api_client) From 032a00390561005dabdaa840c8ae8b36d136ccbd Mon Sep 17 00:00:00 2001 From: Ashutosh K Date: Mon, 12 Jan 2015 18:11:20 +0530 Subject: [PATCH 037/391] CLOUDSTACK-8147: Fixed typo in test case test_redundant_router.py Signed-off-by: SrikanteswaraRao Talluri --- test/integration/component/maint/test_redundant_router.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/component/maint/test_redundant_router.py b/test/integration/component/maint/test_redundant_router.py index 01d635b7e6c..98b0bae1ba0 100644 --- a/test/integration/component/maint/test_redundant_router.py +++ b/test/integration/component/maint/test_redundant_router.py @@ -787,7 +787,7 @@ class TestRVRInternals(cloudstackTestCase): self.apiclient.connection.mgtSvr, 22, self.apiclient.connection.user, - self.apiclient.connction.passwd, + self.apiclient.connection.passwd, backup_router.linklocalip, 'ip addr show eth2', hypervisor=self.hypervisor From b8184d1e74609b0803d234ffb181525c48d28029 Mon Sep 17 00:00:00 2001 From: Ashutosh K Date: Mon, 12 Jan 2015 18:47:29 +0530 Subject: [PATCH 038/391] CLOUDSTACK-8149: Code-refactor - test_VirtualRouter_alerts.py Signed-off-by: SrikanteswaraRao Talluri --- .../component/test_VirtualRouter_alerts.py | 187 +++++++----------- 1 file changed, 70 insertions(+), 117 deletions(-) diff --git a/test/integration/component/test_VirtualRouter_alerts.py b/test/integration/component/test_VirtualRouter_alerts.py index bff12f9c8d3..91a4fcf88f8 100644 --- a/test/integration/component/test_VirtualRouter_alerts.py +++ b/test/integration/component/test_VirtualRouter_alerts.py @@ -16,13 +16,20 @@ # under the License. """ P1 tests for alert receiving from VR on service failure in VR """ -#Import Local Modules -import marvin -from marvin.cloudstackTestCase import * -from marvin.cloudstackAPI import * -from marvin.lib.utils import * -from marvin.lib.base import * -from marvin.lib.common import * +# Import Local Modules +# import marvin +from marvin.cloudstackTestCase import cloudstackTestCase +from marvin.lib.utils import (get_process_status, + cleanup_resources) +from marvin.lib.base import (Account, + ServiceOffering, + VirtualMachine) + +from marvin.lib.common import (list_hosts, + list_routers, + get_zone, + get_domain, + get_template) from nose.plugins.attrib import attr from marvin.codes import FAILED import time @@ -30,76 +37,16 @@ import time _multiprocess_shared_ = True -class Services: - """Test VM Life Cycle Services - """ - - def __init__(self): - self.services = { - - "account": { - "email": "test@test.com", - "firstname": "Test", - "lastname": "User", - "username": "test", - # Random characters are appended in create account to - # ensure unique username generated each time - "password": "password", - }, - "small": - # Create a small virtual machine instance with disk offering - { - "displayname": "testserver", - "username": "root", # VM creds for SSH - "password": "password", - "ssh_port": 22, - "hypervisor": 'XenServer', - "privateport": 22, - "publicport": 22, - "protocol": 'TCP', - }, - "service_offerings": - { - "small": - { - # Small service offering ID to for change VM - # service offering from medium to small - "name": "SmallInstance", - "displaytext": "SmallInstance", - "cpunumber": 1, - "cpuspeed": 100, - "memory": 256, - }, - "big": - { - # Big service offering ID to for change VM - "name": "BigInstance", - "displaytext": "BigInstance", - "cpunumber": 1, - "cpuspeed": 100, - "memory": 512, - } - }, - #Change this - "template": { - "displaytext": "xs", - "name": "xs", - "passwordenabled": False, - }, - "sleep": 60, - "timeout": 10, - #Migrate VM to hostid - "ostype": 'CentOS 5.3 (64-bit)', - # CentOS 5.3 (64-bit) - } - class TestVRServiceFailureAlerting(cloudstackTestCase): + @classmethod def setUpClass(cls): - cls.testClient = super(TestVRServiceFailureAlerting, cls).getClsTestClient() + cls.testClient = super( + TestVRServiceFailureAlerting, + cls).getClsTestClient() cls.api_client = cls.testClient.getApiClient() - cls.services = Services().services + cls.services = cls.testClient.getParsedTestDataConfig() # Get Zone, Domain and templates cls.zone = get_zone(cls.api_client, cls.testClient.getZoneForTests()) @@ -113,7 +60,8 @@ class TestVRServiceFailureAlerting(cloudstackTestCase): ) if template == FAILED: - assert False, "get_template() failed to return template with description %s" % cls.services["ostype"] + assert False, "get_template() failed to return template with \ + description %s" % cls.services["ostype"] # Set Zones and disk offerings ?? cls.services["small"]["zoneid"] = cls.zone.id cls.services["small"]["template"] = template.id @@ -130,7 +78,7 @@ class TestVRServiceFailureAlerting(cloudstackTestCase): cls.services["service_offerings"]["small"] ) - #create a virtual machine + # create a virtual machine cls.virtual_machine = VirtualMachine.create( cls.api_client, cls.services["small"], @@ -146,7 +94,9 @@ class TestVRServiceFailureAlerting(cloudstackTestCase): @classmethod def tearDownClass(cls): - cls.api_client = super(TestVRServiceFailureAlerting, cls).getClsTestClient().getApiClient() + cls.api_client = super( + TestVRServiceFailureAlerting, + cls).getClsTestClient().getApiClient() cleanup_resources(cls.api_client, cls._cleanup) return @@ -157,7 +107,7 @@ class TestVRServiceFailureAlerting(cloudstackTestCase): self.cleanup = [] def tearDown(self): - #Clean up, terminate the created ISOs + # Clean up, terminate the created ISOs cleanup_resources(self.apiclient, self.cleanup) return @@ -165,7 +115,6 @@ class TestVRServiceFailureAlerting(cloudstackTestCase): @attr(tags=["advanced", "basic"]) def test_01_VRServiceFailureAlerting(self): - if self.zone.networktype == "Basic": list_router_response = list_routers( self.apiclient, @@ -184,67 +133,71 @@ class TestVRServiceFailureAlerting(cloudstackTestCase): ) router = list_router_response[0] - hosts = list_hosts( - self.apiclient, - zoneid=router.zoneid, - type='Routing', - state='Up', - id=router.hostid - ) - self.assertEqual( - isinstance(hosts, list), - True, - "Check list host returns a valid list" - ) - host = hosts[0] - self.debug("Router ID: %s, state: %s" % (router.id, router.state)) self.assertEqual( - router.state, - 'Running', - "Check list router response for router state" + router.state, + 'Running', + "Check list router response for router state" ) alertSubject = "Monitoring Service on VR " + router.name if self.hypervisor.lower() in ('vmware', 'hyperv'): result = get_process_status( - self.apiclient.connection.mgtSvr, - 22, - self.apiclient.connection.user, - self.apiclient.connection.passwd, - router.linklocalip, - "service dnsmasq status", - hypervisor=self.hypervisor - ) + self.apiclient.connection.mgtSvr, + 22, + self.apiclient.connection.user, + self.apiclient.connection.passwd, + router.linklocalip, + "service dnsmasq status", + hypervisor=self.hypervisor + ) else: try: - host.user, host.passwd = get_host_credentials(self.config, host.ipaddress) + hosts = list_hosts( + self.apiclient, + zoneid=router.zoneid, + type='Routing', + state='Up', + id=router.hostid + ) + + self.assertEqual( + isinstance(hosts, list), + True, + "Check list host returns a valid list" + ) + + host = hosts[0] result = get_process_status( - host.ipaddress, - 22, - host.user, - host.passwd, - router.linklocalip, - "service apache2 stop" - ) - except KeyError: - self.skipTest("Marvin configuration has no host credentials to check router services") + host.ipaddress, + 22, + self.services["configurableData"]["host"]["username"], + self.services["configurableData"]["host"]["password"], + router.linklocalip, + "service apache2 stop" + ) + + except Exception as e: + raise Exception("Exception raised in getting host\ + credentials: %s " % e) res = str(result) self.debug("apache process status: %s" % res) - time.sleep(2400) #wait for 40 minutes meanwhile monitor service on VR starts the apache service (router.alerts.check.interval default value is 30minutes) + time.sleep(2400) + # wait for 40 minutes meanwhile monitor service on + # VR starts the apache service ( + # router.alerts.check.interval default value is + # 30minutes) qresultset = self.dbclient.execute( - "select id from alert where subject = '%s' ORDER BY id DESC LIMIT 1;" \ - % str(alertSubject) - ) + "select id from alert where subject = '%s' ORDER BY id DESC LIMIT 1;" % + str(alertSubject)) self.assertNotEqual( len(qresultset), 0, "Check DB Query result set" ) - return From 613aa8ea28b7670b13792a638fc493b6fe8ebb21 Mon Sep 17 00:00:00 2001 From: Ashutosh K Date: Tue, 30 Dec 2014 12:55:06 +0530 Subject: [PATCH 039/391] CLOUDSTACK-8135: Fixed cleanup issue in test_escalations_instances.py Signed-off-by: SrikanteswaraRao Talluri --- test/integration/component/test_escalations_instances.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/integration/component/test_escalations_instances.py b/test/integration/component/test_escalations_instances.py index c5afa274ba6..725cac315f2 100644 --- a/test/integration/component/test_escalations_instances.py +++ b/test/integration/component/test_escalations_instances.py @@ -2238,7 +2238,6 @@ class TestInstances(cloudstackTestCase): vm_created, "VM creation failed" ) - self.cleanup.append(vm_created) # Listing all the VMs for a user again for matching zone list_vms_after = VirtualMachine.list( self.userapiclient, From 17e19224398ac38bc9171bd157fb376361630f6d Mon Sep 17 00:00:00 2001 From: "K@clogeny.com" Date: Tue, 30 Dec 2014 15:04:38 +0530 Subject: [PATCH 040/391] CLOUDSTACK-8137: Fixed cleanup issue in sec group tests in test_escalations_instances.py Signed-off-by: SrikanteswaraRao Talluri --- test/integration/component/test_escalations_instances.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/integration/component/test_escalations_instances.py b/test/integration/component/test_escalations_instances.py index 725cac315f2..00d5852bd00 100644 --- a/test/integration/component/test_escalations_instances.py +++ b/test/integration/component/test_escalations_instances.py @@ -3648,7 +3648,6 @@ class TestInstances(cloudstackTestCase): security_group, "Security Group creation failed" ) - self.cleanup.append(security_group) security_groups_list_size = security_groups_list_size + 1 # Listing the networks again From 2ed76b2a55f843053e51d243a1545e92307a0012 Mon Sep 17 00:00:00 2001 From: Ashutosh K Date: Tue, 6 Jan 2015 12:54:49 +0530 Subject: [PATCH 041/391] CLOUDSTACK-8143: Corrected a test in BVT test_routers.py --- test/integration/smoke/test_routers.py | 827 +++++++++++++------------ 1 file changed, 427 insertions(+), 400 deletions(-) diff --git a/test/integration/smoke/test_routers.py b/test/integration/smoke/test_routers.py index 7d5f267b3ef..7940d1244c0 100644 --- a/test/integration/smoke/test_routers.py +++ b/test/integration/smoke/test_routers.py @@ -16,22 +16,35 @@ # under the License. """ BVT tests for routers """ -#Import Local Modules -import marvin +# Import Local Modules from marvin.codes import FAILED -from marvin.cloudstackTestCase import * -from marvin.cloudstackAPI import * -from marvin.sshClient import SshClient -from marvin.lib.utils import * -from marvin.lib.base import * -from marvin.lib.common import * +from marvin.cloudstackTestCase import cloudstackTestCase +from marvin.cloudstackAPI import (stopRouter, + restartNetwork, + startRouter, + rebootRouter) +from marvin.lib.utils import (cleanup_resources, + get_process_status, + get_host_credentials) +from marvin.lib.base import (Account, + ServiceOffering, + VirtualMachine) +from marvin.lib.common import (get_domain, + get_zone, + get_template, + list_hosts, + list_routers, + list_networks, + list_zones, + list_vlan_ipranges) from nose.plugins.attrib import attr -#Import System modules +# Import System modules import time _multiprocess_shared_ = True + class TestRouterServices(cloudstackTestCase): @classmethod @@ -46,47 +59,50 @@ class TestRouterServices(cloudstackTestCase): cls.zone = get_zone(cls.apiclient, testClient.getZoneForTests()) cls.services['mode'] = cls.zone.networktype template = get_template( - cls.apiclient, - cls.zone.id, - cls.services["ostype"] - ) + cls.apiclient, + cls.zone.id, + cls.services["ostype"] + ) if template == FAILED: - cls.fail("get_template() failed to return template with description %s" % cls.services["ostype"]) + cls.fail( + "get_template() failed to return template\ + with description %s" % + cls.services["ostype"]) cls.services["virtual_machine"]["zoneid"] = cls.zone.id - #Create an account, network, VM and IP addresses + # Create an account, network, VM and IP addresses cls.account = Account.create( - cls.apiclient, - cls.services["account"], - domainid=cls.domain.id - ) + cls.apiclient, + cls.services["account"], + domainid=cls.domain.id + ) cls.service_offering = ServiceOffering.create( - cls.apiclient, - cls.services["service_offerings"] - ) + cls.apiclient, + cls.services["service_offerings"] + ) cls.vm_1 = VirtualMachine.create( - cls.apiclient, - cls.services["virtual_machine"], - templateid=template.id, - accountid=cls.account.name, - domainid=cls.account.domainid, - serviceofferingid=cls.service_offering.id - ) + cls.apiclient, + cls.services["virtual_machine"], + templateid=template.id, + accountid=cls.account.name, + domainid=cls.account.domainid, + serviceofferingid=cls.service_offering.id + ) cls.cleanup = [ - cls.account, - cls.service_offering - ] + cls.account, + cls.service_offering + ] return @classmethod def tearDownClass(cls): try: cls.apiclient = super( - TestRouterServices, - cls - ).getClsTestClient().getApiClient() - #Clean up, terminate the created templates + TestRouterServices, + cls + ).getClsTestClient().getApiClient() + # Clean up, terminate the created templates cleanup_resources(cls.apiclient, cls.cleanup) except Exception as e: @@ -98,7 +114,7 @@ class TestRouterServices(cloudstackTestCase): self.hypervisor = self.testClient.getHypervisorInfo() return - @attr(tags = ["advanced", "basic", "sg", "smoke"], required_hardware="true") + @attr(tags=["advanced", "basic", "sg", "smoke"], required_hardware="true") def test_01_router_internal_basic(self): """Test router internal basic zone """ @@ -120,58 +136,61 @@ class TestRouterServices(cloudstackTestCase): domainid=self.account.domainid ) self.assertEqual( - isinstance(list_router_response, list), - True, - "Check list response returns a valid list" - ) + isinstance(list_router_response, list), + True, + "Check list response returns a valid list" + ) router = list_router_response[0] hosts = list_hosts( - self.apiclient, - zoneid=router.zoneid, - type='Routing', - state='Up', - id=router.hostid - ) + self.apiclient, + zoneid=router.zoneid, + type='Routing', + state='Up', + id=router.hostid + ) self.assertEqual( - isinstance(hosts, list), - True, - "Check list host returns a valid list" - ) + isinstance(hosts, list), + True, + "Check list host returns a valid list" + ) host = hosts[0] self.debug("Router ID: %s, state: %s" % (router.id, router.state)) self.assertEqual( - router.state, - 'Running', - "Check list router response for router state" - ) + router.state, + 'Running', + "Check list router response for router state" + ) if self.hypervisor.lower() in ('vmware', 'hyperv'): - result = get_process_status( - self.apiclient.connection.mgtSvr, - 22, - self.apiclient.connection.user, - self.apiclient.connection.passwd, - router.linklocalip, - "service dnsmasq status", - hypervisor=self.hypervisor - ) + result = get_process_status( + self.apiclient.connection.mgtSvr, + 22, + self.apiclient.connection.user, + self.apiclient.connection.passwd, + router.linklocalip, + "service dnsmasq status", + hypervisor=self.hypervisor + ) else: try: - host.user, host.passwd = get_host_credentials(self.config, host.ipaddress) + host.user, host.passwd = get_host_credentials( + self.config, host.ipaddress) result = get_process_status( - host.ipaddress, - 22, - host.user, - host.passwd, - router.linklocalip, - "service dnsmasq status" - ) + host.ipaddress, + 22, + host.user, + host.passwd, + router.linklocalip, + "service dnsmasq status" + ) except KeyError: - self.skipTest("Marvin configuration has no host credentials to check router services") + self.skipTest( + "Marvin configuration has no host credentials to\ + check router services") res = str(result) self.debug("Dnsmasq process status: %s" % res) @@ -182,9 +201,7 @@ class TestRouterServices(cloudstackTestCase): ) return - - - @attr(tags = ["advanced", "advancedns"], required_hardware="false") + @attr(tags=["advanced", "advancedns"], required_hardware="false") def test_02_router_internal_adv(self): """Test router internal advanced zone """ @@ -195,140 +212,145 @@ class TestRouterServices(cloudstackTestCase): # Find router associated with user account list_router_response = list_routers( - self.apiclient, - account=self.account.name, - domainid=self.account.domainid - ) + self.apiclient, + account=self.account.name, + domainid=self.account.domainid + ) self.assertEqual( - isinstance(list_router_response, list), - True, - "Check list response returns a valid list" - ) + isinstance(list_router_response, list), + True, + "Check list response returns a valid list" + ) router = list_router_response[0] hosts = list_hosts( - self.apiclient, - zoneid=router.zoneid, - type='Routing', - state='Up', - id=router.hostid - ) + self.apiclient, + zoneid=router.zoneid, + type='Routing', + state='Up', + id=router.hostid + ) self.assertEqual( - isinstance(hosts, list), - True, - "Check list response returns a valid list" - ) + isinstance(hosts, list), + True, + "Check list response returns a valid list" + ) host = hosts[0] self.debug("Router ID: %s, state: %s" % (router.id, router.state)) self.assertEqual( - router.state, - 'Running', - "Check list router response for router state" - ) + router.state, + 'Running', + "Check list router response for router state" + ) if self.hypervisor.lower() in ('vmware', 'hyperv'): - result = get_process_status( - self.apiclient.connection.mgtSvr, - 22, - self.apiclient.connection.user, - self.apiclient.connection.passwd, - router.linklocalip, - "service dnsmasq status", - hypervisor=self.hypervisor - ) + result = get_process_status( + self.apiclient.connection.mgtSvr, + 22, + self.apiclient.connection.user, + self.apiclient.connection.passwd, + router.linklocalip, + "service dnsmasq status", + hypervisor=self.hypervisor + ) else: try: - host.user, host.passwd = get_host_credentials(self.config, host.ipaddress) + host.user, host.passwd = get_host_credentials( + self.config, host.ipaddress) result = get_process_status( - host.ipaddress, - 22, - host.user, - host.passwd, - router.linklocalip, - "service dnsmasq status" - ) + host.ipaddress, + 22, + host.user, + host.passwd, + router.linklocalip, + "service dnsmasq status" + ) except KeyError: - self.skipTest("Marvin configuration has no host credentials to check router services") + self.skipTest( + "Marvin configuration has no host credentials\ + to check router services") res = str(result) self.debug("Dnsmasq process status: %s" % res) self.assertEqual( - res.count("running"), - 1, - "Check dnsmasq service is running or not" - ) + res.count("running"), + 1, + "Check dnsmasq service is running or not" + ) if self.hypervisor.lower() in ('vmware', 'hyperv'): - result = get_process_status( - self.apiclient.connection.mgtSvr, - 22, - self.apiclient.connection.user, - self.apiclient.connection.passwd, - router.linklocalip, - "service haproxy status", - hypervisor=self.hypervisor - ) + result = get_process_status( + self.apiclient.connection.mgtSvr, + 22, + self.apiclient.connection.user, + self.apiclient.connection.passwd, + router.linklocalip, + "service haproxy status", + hypervisor=self.hypervisor + ) else: try: - host.user, host.passwd = get_host_credentials(self.config, host.ipaddress) + host.user, host.passwd = get_host_credentials( + self.config, host.ipaddress) result = get_process_status( - host.ipaddress, - 22, - host.user, - host.passwd, - router.linklocalip, - "service haproxy status" - ) + host.ipaddress, + 22, + host.user, + host.passwd, + router.linklocalip, + "service haproxy status" + ) except KeyError: - self.skipTest("Marvin configuration has no host credentials to check router services") + self.skipTest( + "Marvin configuration has no host credentials\ + to check router services") res = str(result) self.assertEqual( - res.count("running"), - 1, - "Check haproxy service is running or not" - ) + res.count("running"), + 1, + "Check haproxy service is running or not" + ) self.debug("Haproxy process status: %s" % res) return - @attr(tags = ["advanced", "advancedns", "smoke"], required_hardware="false") + @attr(tags=["advanced", "advancedns", "smoke"], required_hardware="false") def test_03_restart_network_cleanup(self): """Test restart network """ # Validate the following # 1. When cleanup = true, router is destroyed and a new one created - # 2. New router will have new publicIp and linkLocalIp and - # all it's services should resume + # 2. New router should have the same public IP # Find router associated with user account list_router_response = list_routers( - self.apiclient, - account=self.account.name, - domainid=self.account.domainid - ) + self.apiclient, + account=self.account.name, + domainid=self.account.domainid + ) self.assertEqual( - isinstance(list_router_response, list), - True, - "Check list response returns a valid list" - ) + isinstance(list_router_response, list), + True, + "Check list response returns a valid list" + ) router = list_router_response[0] - #Store old values before restart - old_linklocalip = router.linklocalip + # Store old values before restart + old_publicip = router.publicip timeout = 10 # Network should be in Implemented or Setup stage before restart while True: networks = list_networks( - self.apiclient, - account=self.account.name, - domainid=self.account.domainid - ) + self.apiclient, + account=self.account.name, + domainid=self.account.domainid + ) self.assertEqual( - isinstance(networks, list), - True, - "Check list response returns a valid list" - ) + isinstance(networks, list), + True, + "Check list response returns a valid list" + ) network = networks[0] if network.state in ["Implemented", "Setup"]: break @@ -340,9 +362,9 @@ class TestRouterServices(cloudstackTestCase): self.debug( "Restarting network with ID: %s, Network state: %s" % ( - network.id, - network.state - )) + network.id, + network.state + )) cmd = restartNetwork.restartNetworkCmd() cmd.id = network.id cmd.cleanup = True @@ -350,25 +372,25 @@ class TestRouterServices(cloudstackTestCase): # Get router details after restart list_router_response = list_routers( - self.apiclient, - account=self.account.name, - domainid=self.account.domainid - ) + self.apiclient, + account=self.account.name, + domainid=self.account.domainid + ) self.assertEqual( - isinstance(list_router_response, list), - True, - "Check list response returns a valid list" - ) + isinstance(list_router_response, list), + True, + "Check list response returns a valid list" + ) router = list_router_response[0] - self.assertNotEqual( - router.linklocalip, - old_linklocalip, - "Check link-local IP after restart" - ) + self.assertEqual( + router.publicip, + old_publicip, + "Public IP of the router should remain same after network restart" + ) return - @attr(tags = ["advanced", "advancedns", "smoke"], required_hardware="true") + @attr(tags=["advanced", "advancedns", "smoke"], required_hardware="true") def test_04_restart_network_wo_cleanup(self): """Test restart network without cleanup """ @@ -382,15 +404,15 @@ class TestRouterServices(cloudstackTestCase): # Network should be in Implemented or Setup stage before restart while True: networks = list_networks( - self.apiclient, - account=self.account.name, - domainid=self.account.domainid - ) + self.apiclient, + account=self.account.name, + domainid=self.account.domainid + ) self.assertEqual( - isinstance(networks, list), - True, - "Check list response returns a valid list" - ) + isinstance(networks, list), + True, + "Check list response returns a valid list" + ) network = networks[0] if network.state in ["Implemented", "Setup"]: break @@ -402,9 +424,9 @@ class TestRouterServices(cloudstackTestCase): self.debug( "Restarting network with ID: %s, Network state: %s" % ( - network.id, - network.state - )) + network.id, + network.state + )) cmd = restartNetwork.restartNetworkCmd() cmd.id = network.id cmd.cleanup = False @@ -412,78 +434,81 @@ class TestRouterServices(cloudstackTestCase): # Get router details after restart list_router_response = list_routers( - self.apiclient, - account=self.account.name, - domainid=self.account.domainid - ) + self.apiclient, + account=self.account.name, + domainid=self.account.domainid + ) self.assertEqual( - isinstance(list_router_response, list), - True, - "Check list response returns a valid list" - ) + isinstance(list_router_response, list), + True, + "Check list response returns a valid list" + ) router = list_router_response[0] hosts = list_hosts( - self.apiclient, - zoneid=router.zoneid, - type='Routing', - state='Up', - id=router.hostid - ) + self.apiclient, + zoneid=router.zoneid, + type='Routing', + state='Up', + id=router.hostid + ) self.assertEqual( - isinstance(hosts, list), - True, - "Check list response returns a valid list" - ) + isinstance(hosts, list), + True, + "Check list response returns a valid list" + ) host = hosts[0] if self.hypervisor.lower() in ('vmware', 'hyperv'): - res = get_process_status( - self.apiclient.connection.mgtSvr, - 22, - self.apiclient.connection.user, - self.apiclient.connection.passwd, - router.linklocalip, - "uptime", - hypervisor=self.hypervisor - ) + res = get_process_status( + self.apiclient.connection.mgtSvr, + 22, + self.apiclient.connection.user, + self.apiclient.connection.passwd, + router.linklocalip, + "uptime", + hypervisor=self.hypervisor + ) else: try: - host.user, host.passwd = get_host_credentials(self.config, host.ipaddress) + host.user, host.passwd = get_host_credentials( + self.config, host.ipaddress) res = get_process_status( - host.ipaddress, - 22, - host.user, - host.passwd, - router.linklocalip, - "uptime" - ) + host.ipaddress, + 22, + host.user, + host.passwd, + router.linklocalip, + "uptime" + ) except KeyError: - self.skipTest("Marvin configuration has no host credentials to check router services") + self.skipTest( + "Marvin configuration has no host credentials\ + to check router services") # res = 12:37:14 up 1 min, 0 users, load average: 0.61, 0.22, 0.08 # Split result to check the uptime result = res[0].split() self.debug("Router Uptime: %s" % result) self.assertEqual( - str(result[1]), - 'up', - "Check router is running or not" - ) + str(result[1]), + 'up', + "Check router is running or not" + ) if str(result[3]) == "min,": - self.assertEqual( - (int(result[2]) < 3), - True, - "Check uptime is less than 3 mins or not" - ) + self.assertEqual( + (int(result[2]) < 3), + True, + "Check uptime is less than 3 mins or not" + ) else: self.assertEqual( - str(result[3]), - 'sec,', - "Check uptime is in seconds" - ) + str(result[3]), + 'sec,', + "Check uptime is in seconds" + ) return - @attr(tags = ["advanced", "advancedns", "smoke"], required_hardware="false") + @attr(tags=["advanced", "advancedns", "smoke"], required_hardware="false") def test_05_router_basic(self): """Test router basic setup """ @@ -494,62 +519,62 @@ class TestRouterServices(cloudstackTestCase): # 3. router will have a guestIP and a linkLocalIp" list_router_response = list_routers( - self.apiclient, - account=self.account.name, - domainid=self.account.domainid - ) + self.apiclient, + account=self.account.name, + domainid=self.account.domainid + ) self.assertEqual( - isinstance(list_router_response, list), - True, - "Check list response returns a valid list" - ) + isinstance(list_router_response, list), + True, + "Check list response returns a valid list" + ) self.assertNotEqual( - len(list_router_response), - 0, - "Check list router response" - ) + len(list_router_response), + 0, + "Check list router response" + ) for router in list_router_response: self.assertEqual( - router.state, - 'Running', - "Check list router response for router state" - ) + router.state, + 'Running', + "Check list router response for router state" + ) zones = list_zones( - self.apiclient, - id=router.zoneid - ) + self.apiclient, + id=router.zoneid + ) self.assertEqual( - isinstance(zones, list), - True, - "Check list response returns a valid list" - ) + isinstance(zones, list), + True, + "Check list response returns a valid list" + ) zone = zones[0] self.assertEqual( - router.dns1, - zone.dns1, - "Compare DNS1 of router and zone" - ) + router.dns1, + zone.dns1, + "Compare DNS1 of router and zone" + ) self.assertEqual( - router.dns2, - zone.dns2, - "Compare DNS2 of router and zone" - ) + router.dns2, + zone.dns2, + "Compare DNS2 of router and zone" + ) self.assertEqual( - hasattr(router, 'guestipaddress'), - True, - "Check whether router has guest IP field" - ) + hasattr(router, 'guestipaddress'), + True, + "Check whether router has guest IP field" + ) self.assertEqual( - hasattr(router, 'linklocalip'), - True, - "Check whether router has link local IP field" - ) + hasattr(router, 'linklocalip'), + True, + "Check whether router has link local IP field" + ) return - @attr(tags = ["advanced", "advancedns", "smoke"], required_hardware="false") + @attr(tags=["advanced", "advancedns", "smoke"], required_hardware="false") def test_06_router_advanced(self): """Test router advanced setup """ @@ -560,79 +585,79 @@ class TestRouterServices(cloudstackTestCase): # 3. router will have guest,public and linklocal IPs list_router_response = list_routers( - self.apiclient, - account=self.account.name, - domainid=self.account.domainid - ) + self.apiclient, + account=self.account.name, + domainid=self.account.domainid + ) self.assertEqual( - isinstance(list_router_response, list), - True, - "Check list response returns a valid list" - ) + isinstance(list_router_response, list), + True, + "Check list response returns a valid list" + ) self.assertNotEqual( - len(list_router_response), - 0, - "Check list router response" - ) + len(list_router_response), + 0, + "Check list router response" + ) for router in list_router_response: self.assertEqual( - router.state, - 'Running', - "Check list router response for router state" - ) + router.state, + 'Running', + "Check list router response for router state" + ) zones = list_zones( - self.apiclient, - id=router.zoneid - ) + self.apiclient, + id=router.zoneid + ) self.assertEqual( - isinstance(zones, list), - True, - "Check list response returns a valid list" - ) + isinstance(zones, list), + True, + "Check list response returns a valid list" + ) zone = zones[0] self.assertEqual( - router.dns1, - zone.dns1, - "Compare DNS1 of router and zone" - ) + router.dns1, + zone.dns1, + "Compare DNS1 of router and zone" + ) self.assertEqual( - router.dns2, - zone.dns2, - "Compare DNS2 of router and zone" - ) + router.dns2, + zone.dns2, + "Compare DNS2 of router and zone" + ) self.assertEqual( - hasattr(router, 'guestipaddress'), - True, - "Check whether router has guest IP field" - ) + hasattr(router, 'guestipaddress'), + True, + "Check whether router has guest IP field" + ) self.assertEqual( - hasattr(router, 'linklocalip'), - True, - "Check whether router has link local IP field" - ) + hasattr(router, 'linklocalip'), + True, + "Check whether router has link local IP field" + ) - #Fetch corresponding ip ranges information from listVlanIpRanges + # Fetch corresponding ip ranges information from listVlanIpRanges ipranges_response = list_vlan_ipranges( - self.apiclient, - zoneid=router.zoneid - ) + self.apiclient, + zoneid=router.zoneid + ) self.assertEqual( - isinstance(ipranges_response, list), - True, - "Check list response returns a valid list" - ) + isinstance(ipranges_response, list), + True, + "Check list response returns a valid list" + ) iprange = ipranges_response[0] self.assertEqual( - router.gateway, - iprange.gateway, - "Check gateway with that of corresponding IP range" - ) + router.gateway, + iprange.gateway, + "Check gateway with that of corresponding IP range" + ) return - @attr(tags = ["advanced", "advancedns", "smoke"], required_hardware="false") + @attr(tags=["advanced", "advancedns", "smoke"], required_hardware="false") def test_07_stop_router(self): """Test stop router """ @@ -641,41 +666,41 @@ class TestRouterServices(cloudstackTestCase): # 1. listRouter should report the router for the account as stopped list_router_response = list_routers( - self.apiclient, - account=self.account.name, - domainid=self.account.domainid - ) + self.apiclient, + account=self.account.name, + domainid=self.account.domainid + ) self.assertEqual( - isinstance(list_router_response, list), - True, - "Check list response returns a valid list" - ) + isinstance(list_router_response, list), + True, + "Check list response returns a valid list" + ) router = list_router_response[0] self.debug("Stopping the router with ID: %s" % router.id) - #Stop the router + # Stop the router cmd = stopRouter.stopRouterCmd() cmd.id = router.id self.apiclient.stopRouter(cmd) - #List routers to check state of router + # List routers to check state of router router_response = list_routers( - self.apiclient, - id=router.id - ) + self.apiclient, + id=router.id + ) self.assertEqual( - isinstance(router_response, list), - True, - "Check list response returns a valid list" - ) - #List router should have router in stopped state + isinstance(router_response, list), + True, + "Check list response returns a valid list" + ) + # List router should have router in stopped state self.assertEqual( - router_response[0].state, - 'Stopped', - "Check list router response for router state" - ) + router_response[0].state, + 'Stopped', + "Check list router response for router state" + ) return - @attr(tags = ["advanced", "advancedns", "smoke"], required_hardware="false") + @attr(tags=["advanced", "advancedns", "smoke"], required_hardware="false") def test_08_start_router(self): """Test start router """ @@ -684,50 +709,50 @@ class TestRouterServices(cloudstackTestCase): # 1. listRouter should report the router for the account as stopped list_router_response = list_routers( - self.apiclient, - account=self.account.name, - domainid=self.account.domainid - ) + self.apiclient, + account=self.account.name, + domainid=self.account.domainid + ) self.assertEqual( - isinstance(list_router_response, list), - True, - "Check list response returns a valid list" - ) + isinstance(list_router_response, list), + True, + "Check list response returns a valid list" + ) router = list_router_response[0] self.debug("Starting the router with ID: %s" % router.id) - #Start the router + # Start the router cmd = startRouter.startRouterCmd() cmd.id = router.id self.apiclient.startRouter(cmd) - #List routers to check state of router + # List routers to check state of router router_response = list_routers( - self.apiclient, - id=router.id - ) + self.apiclient, + id=router.id + ) self.assertEqual( - isinstance(router_response, list), - True, - "Check list response returns a valid list" - ) - #List router should have router in running state + isinstance(router_response, list), + True, + "Check list response returns a valid list" + ) + # List router should have router in running state self.assertEqual( - router_response[0].state, - 'Running', - "Check list router response for router state" - ) + router_response[0].state, + 'Running', + "Check list router response for router state" + ) return - def verifyRouterResponse(self,router_response,ip): - if (router_response) and (isinstance(router_response, list)) and \ - (router_response[0].state == "Running") and \ - (router_response[0].publicip == ip): - return True - return False + def verifyRouterResponse(self, router_response, ip): + if (router_response) and (isinstance(router_response, list)) and \ + (router_response[0].state == "Running") and \ + (router_response[0].publicip == ip): + return True + return False - @attr(tags = ["advanced", "advancedns", "smoke"], required_hardware="false") + @attr(tags=["advanced", "advancedns", "smoke"], required_hardware="false") def test_09_reboot_router(self): """Test reboot router """ @@ -736,36 +761,38 @@ class TestRouterServices(cloudstackTestCase): # 1. listRouter should report the router for the account as stopped list_router_response = list_routers( - self.apiclient, - account=self.account.name, - domainid=self.account.domainid - ) + self.apiclient, + account=self.account.name, + domainid=self.account.domainid + ) self.assertEqual( - isinstance(list_router_response, list), - True, - "Check list response returns a valid list" - ) + isinstance(list_router_response, list), + True, + "Check list response returns a valid list" + ) router = list_router_response[0] public_ip = router.publicip self.debug("Rebooting the router with ID: %s" % router.id) - #Reboot the router + # Reboot the router cmd = rebootRouter.rebootRouterCmd() cmd.id = router.id self.apiclient.rebootRouter(cmd) - #List routers to check state of router + # List routers to check state of router retries_cnt = 6 while retries_cnt >= 0: router_response = list_routers( - self.apiclient, - id=router.id - ) - if self.verifyRouterResponse(router_response,public_ip): + self.apiclient, + id=router.id + ) + if self.verifyRouterResponse(router_response, public_ip): self.debug("Router is running successfully after reboot") return time.sleep(10) retries_cnt = retries_cnt - 1 - self.fail("Router response after reboot is either is invalid or in stopped state") + self.fail( + "Router response after reboot is either is invalid\ + or in stopped state") return From 6bec69844d196e9b66fffa54f6998d8e45fc27e8 Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Wed, 14 Jan 2015 02:19:10 +0530 Subject: [PATCH 042/391] CLOUDSTACK-8037: Require signed AuthnRequest, adds more security Signed-off-by: Rohit Yadav --- .../api/command/GetServiceProviderMetaDataCmd.java | 12 ++++++++++-- .../org/apache/cloudstack/saml/SAML2AuthManager.java | 2 +- .../apache/cloudstack/saml/SAML2AuthManagerImpl.java | 5 ++++- .../org/apache/cloudstack/utils/auth/SAMLUtils.java | 8 ++++---- 4 files changed, 19 insertions(+), 8 deletions(-) diff --git a/plugins/user-authenticators/saml2/src/org/apache/cloudstack/api/command/GetServiceProviderMetaDataCmd.java b/plugins/user-authenticators/saml2/src/org/apache/cloudstack/api/command/GetServiceProviderMetaDataCmd.java index 194d94f9d29..995405dbdea 100644 --- a/plugins/user-authenticators/saml2/src/org/apache/cloudstack/api/command/GetServiceProviderMetaDataCmd.java +++ b/plugins/user-authenticators/saml2/src/org/apache/cloudstack/api/command/GetServiceProviderMetaDataCmd.java @@ -121,7 +121,7 @@ public class GetServiceProviderMetaDataCmd extends BaseCmd implements APIAuthent SPSSODescriptor spSSODescriptor = new SPSSODescriptorBuilder().buildObject(); spSSODescriptor.setWantAssertionsSigned(true); - spSSODescriptor.setAuthnRequestsSigned(false); + spSSODescriptor.setAuthnRequestsSigned(true); X509KeyInfoGeneratorFactory keyInfoGeneratorFactory = new X509KeyInfoGeneratorFactory(); keyInfoGeneratorFactory.setEmitEntityCertificate(true); @@ -134,7 +134,7 @@ public class GetServiceProviderMetaDataCmd extends BaseCmd implements APIAuthent signKeyDescriptor.setUse(UsageType.SIGNING); BasicX509Credential credential = new BasicX509Credential(); - credential.setEntityCertificate(_samlAuthManager.getSpX509Key()); + credential.setEntityCertificate(_samlAuthManager.getSpX509Certificate()); try { encKeyDescriptor.setKeyInfo(keyInfoGenerator.generate(credential)); signKeyDescriptor.setKeyInfo(keyInfoGenerator.generate(credential)); @@ -148,6 +148,14 @@ public class GetServiceProviderMetaDataCmd extends BaseCmd implements APIAuthent nameIDFormat.setFormat(NameIDType.PERSISTENT); spSSODescriptor.getNameIDFormats().add(nameIDFormat); + NameIDFormat emailNameIDFormat = new NameIDFormatBuilder().buildObject(); + emailNameIDFormat.setFormat(NameIDType.EMAIL); + spSSODescriptor.getNameIDFormats().add(emailNameIDFormat); + + NameIDFormat transientNameIDFormat = new NameIDFormatBuilder().buildObject(); + transientNameIDFormat.setFormat(NameIDType.TRANSIENT); + spSSODescriptor.getNameIDFormats().add(transientNameIDFormat); + AssertionConsumerService assertionConsumerService = new AssertionConsumerServiceBuilder().buildObject(); assertionConsumerService.setIndex(0); assertionConsumerService.setBinding(SAMLConstants.SAML2_REDIRECT_BINDING_URI); diff --git a/plugins/user-authenticators/saml2/src/org/apache/cloudstack/saml/SAML2AuthManager.java b/plugins/user-authenticators/saml2/src/org/apache/cloudstack/saml/SAML2AuthManager.java index 3ee7522a9c6..9c0d4b42fc6 100644 --- a/plugins/user-authenticators/saml2/src/org/apache/cloudstack/saml/SAML2AuthManager.java +++ b/plugins/user-authenticators/saml2/src/org/apache/cloudstack/saml/SAML2AuthManager.java @@ -28,7 +28,7 @@ public interface SAML2AuthManager extends PluggableAPIAuthenticator { public X509Certificate getIdpSigningKey(); public X509Certificate getIdpEncryptionKey(); - public X509Certificate getSpX509Key(); + public X509Certificate getSpX509Certificate(); public KeyPair getSpKeyPair(); public String getSpSingleSignOnUrl(); diff --git a/plugins/user-authenticators/saml2/src/org/apache/cloudstack/saml/SAML2AuthManagerImpl.java b/plugins/user-authenticators/saml2/src/org/apache/cloudstack/saml/SAML2AuthManagerImpl.java index f175081f60a..36c9da5e1e0 100644 --- a/plugins/user-authenticators/saml2/src/org/apache/cloudstack/saml/SAML2AuthManagerImpl.java +++ b/plugins/user-authenticators/saml2/src/org/apache/cloudstack/saml/SAML2AuthManagerImpl.java @@ -202,6 +202,9 @@ public class SAML2AuthManagerImpl extends AdapterBase implements SAML2AuthManage s_logger.error("SAML2 Authentication may be unavailable"); } catch (ConfigurationException | FactoryConfigurationError e) { s_logger.error("OpenSAML bootstrapping failed: error: " + e.getMessage()); + } catch (NullPointerException e) { + s_logger.error("Unable to setup SAML Auth Plugin due to NullPointerException" + + " please check the SAML IDP metadata URL and entity ID in global settings: " + e.getMessage()); } if (this.idpSingleLogOutUrl == null || this.idpSingleSignOnUrl == null) { @@ -259,7 +262,7 @@ public class SAML2AuthManagerImpl extends AdapterBase implements SAML2AuthManage return Boolean.valueOf(_configDao.getValue(Config.SAMLIsPluginEnabled.key())); } - public X509Certificate getSpX509Key() { + public X509Certificate getSpX509Certificate() { return spX509Key; } diff --git a/utils/src/org/apache/cloudstack/utils/auth/SAMLUtils.java b/utils/src/org/apache/cloudstack/utils/auth/SAMLUtils.java index bb4af3a57d5..c75422f8774 100644 --- a/utils/src/org/apache/cloudstack/utils/auth/SAMLUtils.java +++ b/utils/src/org/apache/cloudstack/utils/auth/SAMLUtils.java @@ -148,7 +148,7 @@ public class SAMLUtils { RequestedAuthnContextBuilder requestedAuthnContextBuilder = new RequestedAuthnContextBuilder(); RequestedAuthnContext requestedAuthnContext = requestedAuthnContextBuilder.buildObject(); requestedAuthnContext - .setComparison(AuthnContextComparisonTypeEnumeration.MINIMUM); + .setComparison(AuthnContextComparisonTypeEnumeration.EXACT); requestedAuthnContext.getAuthnContextClassRefs().add( authnContextClassRef); @@ -166,7 +166,7 @@ public class SAMLUtils { authnRequest.setAssertionConsumerServiceURL(consumerUrl); authnRequest.setProviderName(spId); authnRequest.setNameIDPolicy(nameIdPolicy); - //authnRequest.setRequestedAuthnContext(requestedAuthnContext); + authnRequest.setRequestedAuthnContext(requestedAuthnContext); return authnRequest; } @@ -315,8 +315,8 @@ public class SAMLUtils { public static X509Certificate generateRandomX509Certificate(KeyPair keyPair) throws NoSuchAlgorithmException, NoSuchProviderException, CertificateEncodingException, SignatureException, InvalidKeyException { Date validityBeginDate = new Date(System.currentTimeMillis() - 24 * 60 * 60 * 1000); - Date validityEndDate = new Date(System.currentTimeMillis() + 365 * 24 * 60 * 60 * 1000); - X500Principal dnName = new X500Principal("CN=Apache CloudStack"); + Date validityEndDate = new Date(System.currentTimeMillis() + 3 * 365 * 24 * 60 * 60 * 1000); + X500Principal dnName = new X500Principal("CN=ApacheCloudStack"); X509V1CertificateGenerator certGen = new X509V1CertificateGenerator(); certGen.setSerialNumber(BigInteger.valueOf(System.currentTimeMillis())); certGen.setSubjectDN(dnName); From 921ad057def3015cda9d9f5861c9be29a88b148e Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Wed, 14 Jan 2015 02:33:34 +0530 Subject: [PATCH 043/391] CLOUDSTACK-8155: Remove unwanted whitespaces from json response This removes extra whitespaces from the JSON serialized response. After the fix, tested to work with: - Present UI - CloudMonkey - Old buggy json parsers Signed-off-by: Rohit Yadav --- .../api/response/ApiResponseSerializer.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/server/src/com/cloud/api/response/ApiResponseSerializer.java b/server/src/com/cloud/api/response/ApiResponseSerializer.java index a32bd5604a9..19d26710540 100644 --- a/server/src/com/cloud/api/response/ApiResponseSerializer.java +++ b/server/src/com/cloud/api/response/ApiResponseSerializer.java @@ -79,13 +79,13 @@ public class ApiResponseSerializer { StringBuilder sb = new StringBuilder(); - sb.append("{ \"").append(result.getResponseName()).append("\" : "); + sb.append("{\"").append(result.getResponseName()).append("\":"); if (result instanceof ListResponse) { List responses = ((ListResponse)result).getResponses(); Integer count = ((ListResponse)result).getCount(); boolean nonZeroCount = (count != null && count.longValue() != 0); if (nonZeroCount) { - sb.append("{ \"").append(ApiConstants.COUNT).append("\":").append(count); + sb.append("{\"").append(ApiConstants.COUNT).append("\":").append(count); } if ((responses != null) && !responses.isEmpty()) { @@ -93,24 +93,24 @@ public class ApiResponseSerializer { jsonStr = unescape(jsonStr); if (nonZeroCount) { - sb.append(" ,\"").append(responses.get(0).getObjectName()).append("\" : [ ").append(jsonStr); + sb.append(",\"").append(responses.get(0).getObjectName()).append("\":[").append(jsonStr); } for (int i = 1; i < ((ListResponse)result).getResponses().size(); i++) { jsonStr = gson.toJson(responses.get(i)); jsonStr = unescape(jsonStr); - sb.append(", ").append(jsonStr); + sb.append(",").append(jsonStr); } - sb.append(" ] }"); + sb.append("]}"); } else { if (!nonZeroCount) { sb.append("{"); } - sb.append(" }"); + sb.append("}"); } } else if (result instanceof SuccessResponse) { - sb.append("{ \"success\" : \"").append(((SuccessResponse)result).getSuccess()).append("\"} "); + sb.append("{\"success\":\"").append(((SuccessResponse)result).getSuccess()).append("\"}"); } else if (result instanceof ExceptionResponse) { String jsonErrorText = gson.toJson(result); jsonErrorText = unescape(jsonErrorText); @@ -122,13 +122,13 @@ public class ApiResponseSerializer { if (result instanceof AsyncJobResponse || result instanceof CreateCmdResponse || result instanceof AuthenticationCmdResponse) { sb.append(jsonStr); } else { - sb.append(" { \"").append(result.getObjectName()).append("\" : ").append(jsonStr).append(" } "); + sb.append("{\"").append(result.getObjectName()).append("\":").append(jsonStr).append("}"); } } else { - sb.append("{ }"); + sb.append("{}"); } } - sb.append(" }"); + sb.append("}"); return sb.toString(); } return null; From 5df6ee4f7c5b8715627131465c9507e75d111419 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Moser?= Date: Tue, 13 Jan 2015 16:23:31 +0100 Subject: [PATCH 044/391] CLOUDSTACK-8154: Fix typo in response name string: there is an extra 's' MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: René Moser (cherry picked from commit c7ebd13d02948fd2ca2d6e4245127e01a2f7579e) Signed-off-by: Rohit Yadav --- .../apache/cloudstack/api/command/user/iso/DeleteIsoCmd.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/org/apache/cloudstack/api/command/user/iso/DeleteIsoCmd.java b/api/src/org/apache/cloudstack/api/command/user/iso/DeleteIsoCmd.java index b07b08dbf84..6101e9f9c06 100644 --- a/api/src/org/apache/cloudstack/api/command/user/iso/DeleteIsoCmd.java +++ b/api/src/org/apache/cloudstack/api/command/user/iso/DeleteIsoCmd.java @@ -38,7 +38,7 @@ import com.cloud.user.Account; requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) public class DeleteIsoCmd extends BaseAsyncCmd { public static final Logger s_logger = Logger.getLogger(DeleteIsoCmd.class.getName()); - private static final String s_name = "deleteisosresponse"; + private static final String s_name = "deleteisoresponse"; ///////////////////////////////////////////////////// //////////////// API parameters ///////////////////// From 9a677595fa5cc351d679e20243c18faab7742c25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Moser?= Date: Tue, 13 Jan 2015 16:34:04 +0100 Subject: [PATCH 045/391] CLOUDSTACK-8154: fix UI for API change MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: René Moser (cherry picked from commit 07d96aab843a3f191e3cfbff57560d821938076a) Signed-off-by: Rohit Yadav --- ui/scripts/templates.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/scripts/templates.js b/ui/scripts/templates.js index dc0f055bf01..b95bb7dd759 100644 --- a/ui/scripts/templates.js +++ b/ui/scripts/templates.js @@ -2192,7 +2192,7 @@ dataType: "json", async: true, success: function(json) { - var jid = json.deleteisosresponse.jobid; + var jid = json.deleteisoresponse.jobid; args.response.success({ _custom: { jobId: jid From 162c5af6f8ede5adcabcdc026a08028f4b442f60 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Ritschard Date: Wed, 14 Jan 2015 11:27:35 +0100 Subject: [PATCH 046/391] Use constant-time comparison functions when checking signatures This limits the likeliness of timing attacks against the API. See http://codahale.com/a-lesson-in-timing-attacks/ for the full rationale. Signed-off-by: Rohit Yadav Conflicts: server/src/com/cloud/api/ApiServer.java server/src/com/cloud/user/AccountManagerImpl.java (cherry picked from commit 9b4e39e837af498599859c4a6687eb8bf9f8ad89) Signed-off-by: Rohit Yadav --- server/src/com/cloud/api/ApiServer.java | 4 ++- .../com/cloud/api/ConstantTimeComparator.java | 36 +++++++++++++++++++ .../cloud/servlet/ConsoleProxyServlet.java | 3 +- .../com/cloud/user/AccountManagerImpl.java | 4 ++- 4 files changed, 44 insertions(+), 3 deletions(-) create mode 100644 server/src/com/cloud/api/ConstantTimeComparator.java diff --git a/server/src/com/cloud/api/ApiServer.java b/server/src/com/cloud/api/ApiServer.java index f35bd9dcaf7..785a82213ee 100755 --- a/server/src/com/cloud/api/ApiServer.java +++ b/server/src/com/cloud/api/ApiServer.java @@ -908,9 +908,11 @@ public class ApiServer extends ManagerBase implements HttpRequestHandler, ApiSer final SecretKeySpec keySpec = new SecretKeySpec(secretKey.getBytes(), "HmacSHA1"); mac.init(keySpec); mac.update(unsignedRequest.getBytes()); + final byte[] encryptedBytes = mac.doFinal(); final String computedSignature = Base64.encodeBase64String(encryptedBytes); - final boolean equalSig = signature.equals(computedSignature); + final boolean equalSig = ConstantTimeComparator.compareStrings(signature, computedSignature); + if (!equalSig) { s_logger.info("User signature: " + signature + " is not equaled to computed signature: " + computedSignature); } else { diff --git a/server/src/com/cloud/api/ConstantTimeComparator.java b/server/src/com/cloud/api/ConstantTimeComparator.java new file mode 100644 index 00000000000..4612eeeb5bb --- /dev/null +++ b/server/src/com/cloud/api/ConstantTimeComparator.java @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.api; + +public class ConstantTimeComparator { + + public static boolean compareBytes(byte[] b1, byte[] b2) { + if (b1.length != b2.length) { + return false; + } + + int result = 0; + for (int i = 0; i < b1.length; i++) { + result |= b1[i] ^ b2[i]; + } + return result == 0; + } + + public static boolean compareStrings(String s1, String s2) { + return compareBytes(s1.getBytes(), s2.getBytes()); + } +} diff --git a/server/src/com/cloud/servlet/ConsoleProxyServlet.java b/server/src/com/cloud/servlet/ConsoleProxyServlet.java index bf021ea4030..b6a8d63b245 100644 --- a/server/src/com/cloud/servlet/ConsoleProxyServlet.java +++ b/server/src/com/cloud/servlet/ConsoleProxyServlet.java @@ -45,6 +45,7 @@ import com.google.gson.GsonBuilder; import org.apache.cloudstack.framework.security.keys.KeysManager; +import com.cloud.api.ConstantTimeComparator; import com.cloud.exception.PermissionDeniedException; import com.cloud.host.HostVO; import com.cloud.server.ManagementServer; @@ -653,7 +654,7 @@ public class ConsoleProxyServlet extends HttpServlet { mac.update(unsignedRequest.getBytes()); byte[] encryptedBytes = mac.doFinal(); String computedSignature = Base64.encodeBase64String(encryptedBytes); - boolean equalSig = signature.equals(computedSignature); + boolean equalSig = ConstantTimeComparator.compareStrings(signature, computedSignature); if (!equalSig) { s_logger.debug("User signature: " + signature + " is not equaled to computed signature: " + computedSignature); } diff --git a/server/src/com/cloud/user/AccountManagerImpl.java b/server/src/com/cloud/user/AccountManagerImpl.java index 3c00f91cd75..44c687fd436 100755 --- a/server/src/com/cloud/user/AccountManagerImpl.java +++ b/server/src/com/cloud/user/AccountManagerImpl.java @@ -62,6 +62,7 @@ import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.cloudstack.region.gslb.GlobalLoadBalancerRuleDao; import com.cloud.api.ApiDBUtils; +import com.cloud.api.ConstantTimeComparator; import com.cloud.api.query.vo.ControlledViewEntity; import com.cloud.configuration.Config; import com.cloud.configuration.ConfigurationManager; @@ -485,6 +486,7 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M @Override public void checkAccess(Account caller, AccessType accessType, boolean sameOwner, String apiName, ControlledEntity... entities) { + //check for the same owner Long ownerId = null; ControlledEntity prevEntity = null; @@ -2058,7 +2060,7 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M mac.update(unsignedRequest.getBytes()); byte[] encryptedBytes = mac.doFinal(); String computedSignature = new String(Base64.encodeBase64(encryptedBytes)); - boolean equalSig = signature.equals(computedSignature); + boolean equalSig = ConstantTimeComparator.compareStrings(signature, computedSignature); if (!equalSig) { s_logger.info("User signature: " + signature + " is not equaled to computed signature: " + computedSignature); } else { From 5ea74147460550ea0fdbcdf1139fb3f104006458 Mon Sep 17 00:00:00 2001 From: Pierre-Yves Ritschard Date: Wed, 14 Jan 2015 12:14:00 +0100 Subject: [PATCH 047/391] move ConstantTimeComparator to utils (cherry picked from commit b2393c31ed8f689e45227f12371fc042c9dbd0e4) Signed-off-by: Rohit Yadav --- server/src/com/cloud/api/ApiServer.java | 1 + server/src/com/cloud/servlet/ConsoleProxyServlet.java | 2 +- server/src/com/cloud/user/AccountManagerImpl.java | 2 +- .../src/com/cloud/utils}/ConstantTimeComparator.java | 5 ++++- 4 files changed, 7 insertions(+), 3 deletions(-) rename {server/src/com/cloud/api => utils/src/com/cloud/utils}/ConstantTimeComparator.java (97%) diff --git a/server/src/com/cloud/api/ApiServer.java b/server/src/com/cloud/api/ApiServer.java index 785a82213ee..8821d0f90a3 100755 --- a/server/src/com/cloud/api/ApiServer.java +++ b/server/src/com/cloud/api/ApiServer.java @@ -40,6 +40,7 @@ import com.cloud.user.DomainManager; import com.cloud.user.User; import com.cloud.user.UserAccount; import com.cloud.user.UserVO; +import com.cloud.utils.ConstantTimeComparator; import com.cloud.utils.HttpUtils; import com.cloud.utils.NumbersUtil; import com.cloud.utils.Pair; diff --git a/server/src/com/cloud/servlet/ConsoleProxyServlet.java b/server/src/com/cloud/servlet/ConsoleProxyServlet.java index b6a8d63b245..4694a471b62 100644 --- a/server/src/com/cloud/servlet/ConsoleProxyServlet.java +++ b/server/src/com/cloud/servlet/ConsoleProxyServlet.java @@ -45,7 +45,6 @@ import com.google.gson.GsonBuilder; import org.apache.cloudstack.framework.security.keys.KeysManager; -import com.cloud.api.ConstantTimeComparator; import com.cloud.exception.PermissionDeniedException; import com.cloud.host.HostVO; import com.cloud.server.ManagementServer; @@ -54,6 +53,7 @@ import com.cloud.user.Account; import com.cloud.user.AccountManager; import com.cloud.user.User; import com.cloud.uservm.UserVm; +import com.cloud.utils.ConstantTimeComparator; import com.cloud.utils.Pair; import com.cloud.utils.Ternary; import com.cloud.utils.db.EntityManager; diff --git a/server/src/com/cloud/user/AccountManagerImpl.java b/server/src/com/cloud/user/AccountManagerImpl.java index 44c687fd436..6e9e0d72928 100755 --- a/server/src/com/cloud/user/AccountManagerImpl.java +++ b/server/src/com/cloud/user/AccountManagerImpl.java @@ -62,7 +62,6 @@ import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.cloudstack.region.gslb.GlobalLoadBalancerRuleDao; import com.cloud.api.ApiDBUtils; -import com.cloud.api.ConstantTimeComparator; import com.cloud.api.query.vo.ControlledViewEntity; import com.cloud.configuration.Config; import com.cloud.configuration.ConfigurationManager; @@ -133,6 +132,7 @@ import com.cloud.user.Account.State; import com.cloud.user.dao.AccountDao; import com.cloud.user.dao.UserAccountDao; import com.cloud.user.dao.UserDao; +import com.cloud.utils.ConstantTimeComparator; import com.cloud.utils.NumbersUtil; import com.cloud.utils.Pair; import com.cloud.utils.Ternary; diff --git a/server/src/com/cloud/api/ConstantTimeComparator.java b/utils/src/com/cloud/utils/ConstantTimeComparator.java similarity index 97% rename from server/src/com/cloud/api/ConstantTimeComparator.java rename to utils/src/com/cloud/utils/ConstantTimeComparator.java index 4612eeeb5bb..4d4a595309a 100644 --- a/server/src/com/cloud/api/ConstantTimeComparator.java +++ b/utils/src/com/cloud/utils/ConstantTimeComparator.java @@ -1,3 +1,4 @@ +// // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information @@ -14,7 +15,9 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -package com.cloud.api; +// + +package com.cloud.utils; public class ConstantTimeComparator { From 7885a6c4efe13a07b3966ef8684152dda5108f8b Mon Sep 17 00:00:00 2001 From: Erik Weber Date: Wed, 14 Jan 2015 14:52:47 +0100 Subject: [PATCH 048/391] Add absolute schema references to support MySQL 5.6 better (cherry picked from commit 396936ea5e84d079394664fd1d0596a94e159855) Signed-off-by: Rohit Yadav --- setup/db/create-schema-premium.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/db/create-schema-premium.sql b/setup/db/create-schema-premium.sql index 2f86c0b9ab5..a8feff448a9 100644 --- a/setup/db/create-schema-premium.sql +++ b/setup/db/create-schema-premium.sql @@ -296,7 +296,7 @@ CREATE TABLE `cloud`.`netapp_volume` ( `password` varchar(200) COMMENT 'password', `round_robin_marker` int COMMENT 'This marks the volume to be picked up for lun creation, RR fashion', PRIMARY KEY (`id`), - CONSTRAINT `fk_netapp_volume__pool_id` FOREIGN KEY `fk_netapp_volume__pool_id` (`pool_id`) REFERENCES `netapp_pool` (`id`) ON DELETE CASCADE, + CONSTRAINT `fk_netapp_volume__pool_id` FOREIGN KEY `fk_netapp_volume__pool_id` (`pool_id`) REFERENCES `cloud`.`netapp_pool` (`id`) ON DELETE CASCADE, INDEX `i_netapp_volume__pool_id`(`pool_id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; @@ -315,7 +315,7 @@ CREATE TABLE `cloud`.`netapp_lun` ( `size` bigint NOT NULL COMMENT 'lun size', `volume_id` bigint unsigned NOT NULL COMMENT 'parent volume id', PRIMARY KEY (`id`), - CONSTRAINT `fk_netapp_lun__volume_id` FOREIGN KEY `fk_netapp_lun__volume_id` (`volume_id`) REFERENCES `netapp_volume` (`id`) ON DELETE CASCADE, + CONSTRAINT `fk_netapp_lun__volume_id` FOREIGN KEY `fk_netapp_lun__volume_id` (`volume_id`) REFERENCES `cloud`.`netapp_volume` (`id`) ON DELETE CASCADE, INDEX `i_netapp_lun__volume_id`(`volume_id`), INDEX `i_netapp_lun__lun_name`(`lun_name`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; From b99d950dd31420ffb06d6c910a075ec691ae2e67 Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Thu, 15 Jan 2015 15:31:27 +0530 Subject: [PATCH 049/391] appliance: fix Debian iso url, fixes failing Jenkins builds Signed-off-by: Rohit Yadav --- .../appliance/definitions/systemvm64template/definition.rb | 6 +++--- tools/appliance/definitions/systemvmtemplate/definition.rb | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/appliance/definitions/systemvm64template/definition.rb b/tools/appliance/definitions/systemvm64template/definition.rb index 8dbf7241679..35406cced3c 100644 --- a/tools/appliance/definitions/systemvm64template/definition.rb +++ b/tools/appliance/definitions/systemvm64template/definition.rb @@ -10,9 +10,9 @@ Veewee::Definition.declare({ :memory_size=> '256', :disk_size => '2500', :disk_format => 'VDI', :hostiocache => 'off', :os_type_id => 'Debian_64', - :iso_file => "debian-7.7.0-amd64-netinst.iso", - :iso_src => "http://ftp.acc.umu.se/debian-cd/7.7.0/amd64/iso-cd/debian-7.7.0-amd64-netinst.iso", - :iso_md5 => "0b31bccccb048d20b551f70830bb7ad0", + :iso_file => "debian-7.8.0-amd64-netinst.iso", + :iso_src => "http://cdimage.debian.org/debian-cd/7.8.0/amd64/iso-cd/debian-7.8.0-amd64-netinst.iso", + :iso_md5 => "a91fba5001cf0fbccb44a7ae38c63b6e", :iso_download_timeout => "1000", :boot_wait => "10", :boot_cmd_sequence => [ '', diff --git a/tools/appliance/definitions/systemvmtemplate/definition.rb b/tools/appliance/definitions/systemvmtemplate/definition.rb index 91514bf454a..7d2973eb3cf 100644 --- a/tools/appliance/definitions/systemvmtemplate/definition.rb +++ b/tools/appliance/definitions/systemvmtemplate/definition.rb @@ -10,9 +10,9 @@ Veewee::Definition.declare({ :memory_size=> '256', :disk_size => '2500', :disk_format => 'VDI', :hostiocache => 'off', :os_type_id => 'Debian', - :iso_file => "debian-7.7.0-i386-netinst.iso", - :iso_src => "http://ftp.acc.umu.se/debian-cd/7.7.0/i386/iso-cd/debian-7.7.0-i386-netinst.iso", - :iso_md5 => "76d512c44a9b7eca53ea2812ad5ac36f", + :iso_file => "debian-7.8.0-i386-netinst.iso", + :iso_src => "http://cdimage.debian.org/debian-cd/7.8.0/i386/iso-cd/debian-7.8.0-i386-netinst.iso", + :iso_md5 => "0d2f88d23a9d5945f5bc0276830c7d2c", :iso_download_timeout => "1000", :boot_wait => "10", :boot_cmd_sequence => [ '', From 188924751ed87a01541a094e03e958cd8d01653b Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Thu, 2 Oct 2014 11:58:20 +0200 Subject: [PATCH 050/391] CLOUDSTACK-7219: Fix NPE, log warning when config item is missing from scope - Cherry picked from Daan's fix 63fbd16dd11388bd93cdbec4ea7fe6de37aa7fc5 - Added another check if configDepot returned null - Removed developer prefill values Signed-off-by: Rohit Yadav --- developer/developer-prefill.sql | 16 ---------------- .../com/cloud/server/ManagementServerImpl.java | 15 ++++++++++++--- 2 files changed, 12 insertions(+), 19 deletions(-) diff --git a/developer/developer-prefill.sql b/developer/developer-prefill.sql index c2d4f4357d8..89d9c7ebe28 100644 --- a/developer/developer-prefill.sql +++ b/developer/developer-prefill.sql @@ -62,22 +62,6 @@ INSERT INTO `cloud`.`configuration` (category, instance, component, name, value) VALUES ('Advanced', 'DEFAULT', 'management-server', 'expunge.interval', '60'); -INSERT INTO `cloud`.`configuration` (category, instance, component, name, value) - VALUES ('Advanced', 'DEFAULT', 'management-server', - 'cluster.cpu.allocated.capacity.disablethreshold', '0.95'); - -INSERT INTO `cloud`.`configuration` (category, instance, component, name, value) - VALUES ('Advanced', 'DEFAULT', 'management-server', - 'cluster.memory.allocated.capacity.disablethreshold ', '0.95'); - -INSERT INTO `cloud`.`configuration` (category, instance, component, name, value) - VALUES ('Advanced', 'DEFAULT', 'management-server', - 'pool.storage.allocated.capacity.disablethreshold ', '0.95'); - -INSERT INTO `cloud`.`configuration` (category, instance, component, name, value) - VALUES ('Advanced', 'DEFAULT', 'management-server', - 'pool.storage.capacity.disablethreshold ', '0.95'); - -- Add developer configuration entry; allows management server to be run as a user other than "cloud" INSERT INTO `cloud`.`configuration` (category, instance, component, name, value) VALUES ('Advanced', 'DEFAULT', 'management-server', diff --git a/server/src/com/cloud/server/ManagementServerImpl.java b/server/src/com/cloud/server/ManagementServerImpl.java index deb24245ebd..85ce413d96a 100755 --- a/server/src/com/cloud/server/ManagementServerImpl.java +++ b/server/src/com/cloud/server/ManagementServerImpl.java @@ -1709,9 +1709,18 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe List configVOList = new ArrayList(); for (ConfigurationVO param : result.first()) { ConfigurationVO configVo = _configDao.findByName(param.getName()); - configVo.setValue(_configDepot.get(param.getName()).valueIn(id).toString()); - configVOList.add(configVo); - } + if (configVo != null) { + ConfigKey key = _configDepot.get(param.getName()); + if (key != null) { + configVo.setValue(key.valueIn(id).toString()); + configVOList.add(configVo); + } else { + s_logger.warn("ConfigDepot could not find parameter " + param.getName() + " for scope " + scope); + } + } else { + s_logger.warn("Configuration item " + param.getName() + " not found in " + scope); + } + } return new Pair, Integer>(configVOList, configVOList.size()); } From 59b2cbafa1f0c3033655112d348bb9523e0e8ef0 Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Thu, 15 Jan 2015 20:05:45 +0530 Subject: [PATCH 051/391] developer: re-add deleted configuration Signed-off-by: Rohit Yadav --- developer/developer-prefill.sql | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/developer/developer-prefill.sql b/developer/developer-prefill.sql index 89d9c7ebe28..27b36e78581 100644 --- a/developer/developer-prefill.sql +++ b/developer/developer-prefill.sql @@ -62,6 +62,22 @@ INSERT INTO `cloud`.`configuration` (category, instance, component, name, value) VALUES ('Advanced', 'DEFAULT', 'management-server', 'expunge.interval', '60'); +INSERT INTO `cloud`.`configuration` (category, instance, component, name, value) + VALUES ('Advanced', 'DEFAULT', 'management-server', + 'cluster.cpu.allocated.capacity.disablethreshold', '0.95'); + +INSERT INTO `cloud`.`configuration` (category, instance, component, name, value) + VALUES ('Advanced', 'DEFAULT', 'management-server', + 'cluster.memory.allocated.capacity.disablethreshold', '0.95'); + +INSERT INTO `cloud`.`configuration` (category, instance, component, name, value) + VALUES ('Advanced', 'DEFAULT', 'management-server', + 'pool.storage.allocated.capacity.disablethreshold', '0.95'); + +INSERT INTO `cloud`.`configuration` (category, instance, component, name, value) + VALUES ('Advanced', 'DEFAULT', 'management-server', + 'pool.storage.capacity.disablethreshold', '0.95'); + -- Add developer configuration entry; allows management server to be run as a user other than "cloud" INSERT INTO `cloud`.`configuration` (category, instance, component, name, value) VALUES ('Advanced', 'DEFAULT', 'management-server', From e09308a1d469fd8a5120156f434bfee879dc2e67 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Fri, 16 Jan 2015 10:56:25 +0100 Subject: [PATCH 052/391] CLOUDSTACK-7219: Cannot display Cluster Settings for missing commit 95e41fdf0da50c165a9317847058fce4efeddbcf (cherry picked from commit 71103772b7502ff3a9f9af101652fe431cb8f630) --- .../cloud/agent/api/to/VirtualMachineTO.java | 9 ++++++ .../com/cloud/hypervisor/XenServerGuru.java | 29 ++++++++++++++++++- .../resource/CitrixResourceBase.java | 17 +++++------ 3 files changed, 45 insertions(+), 10 deletions(-) diff --git a/api/src/com/cloud/agent/api/to/VirtualMachineTO.java b/api/src/com/cloud/agent/api/to/VirtualMachineTO.java index 4b36df5827f..3e508eea315 100644 --- a/api/src/com/cloud/agent/api/to/VirtualMachineTO.java +++ b/api/src/com/cloud/agent/api/to/VirtualMachineTO.java @@ -61,6 +61,7 @@ public class VirtualMachineTO { DiskTO[] disks; NicTO[] nics; GPUDeviceTO gpuDevice; + Integer vcpuMaxLimit; public VirtualMachineTO(long id, String instanceName, VirtualMachine.Type type, int cpus, Integer speed, long minRam, long maxRam, BootloaderType bootloader, String os, boolean enableHA, boolean limitCpuUse, String vncPassword) { @@ -283,4 +284,12 @@ public class VirtualMachineTO { this.platformEmulator = platformEmulator; } + public Integer getVcpuMaxLimit() { + return vcpuMaxLimit; + } + + public void setVcpuMaxLimit(Integer vcpuMaxLimit) { + this.vcpuMaxLimit = vcpuMaxLimit; + } + } diff --git a/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/XenServerGuru.java b/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/XenServerGuru.java index 89c204638b6..b17048b435c 100644 --- a/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/XenServerGuru.java +++ b/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/XenServerGuru.java @@ -26,12 +26,14 @@ import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; +import org.apache.cloudstack.framework.config.Configurable; import org.apache.cloudstack.hypervisor.xenserver.XenserverConfigs; import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.command.DettachCommand; import org.apache.cloudstack.storage.command.StorageSubSystemCommand; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.framework.config.ConfigKey; import com.cloud.agent.api.Command; import com.cloud.agent.api.to.DataObjectType; @@ -51,11 +53,13 @@ import com.cloud.storage.dao.GuestOSHypervisorDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.template.VirtualMachineTemplate.BootloaderType; import com.cloud.utils.Pair; +import com.cloud.vm.UserVmVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineProfile; +import com.cloud.vm.dao.UserVmDao; @Local(value = HypervisorGuru.class) -public class XenServerGuru extends HypervisorGuruBase implements HypervisorGuru { +public class XenServerGuru extends HypervisorGuruBase implements HypervisorGuru, Configurable { @Inject GuestOSDao _guestOsDao; @Inject @@ -70,6 +74,11 @@ public class XenServerGuru extends HypervisorGuruBase implements HypervisorGuru PrimaryDataStoreDao _storagePoolDao; @Inject VolumeDataFactory _volFactory; + @Inject + UserVmDao _userVmDao; + + static final ConfigKey MaxNumberOfVCPUSPerVM = new ConfigKey("Advanced", Integer.class, "xen.vm.vcpu.max", "16", + "Maximum number of VCPUs that VM can get in XenServer.", true, ConfigKey.Scope.Cluster); protected XenServerGuru() { super(); @@ -87,6 +96,14 @@ public class XenServerGuru extends HypervisorGuruBase implements HypervisorGuru bt = vm.getBootLoaderType(); } VirtualMachineTO to = toVirtualMachineTO(vm); + UserVmVO userVmVO = _userVmDao.findById(vm.getId()); + if (userVmVO != null) { + HostVO host = hostDao.findById(userVmVO.getHostId()); + if (host != null) { + to.setVcpuMaxLimit(MaxNumberOfVCPUSPerVM.valueIn(host.getClusterId())); + } + } + to.setBootloader(bt); // Determine the VM's OS description @@ -176,4 +193,14 @@ public class XenServerGuru extends HypervisorGuruBase implements HypervisorGuru } return new Pair(Boolean.FALSE, new Long(hostId)); } + + @Override + public String getConfigComponentName() { + return XenServerGuru.class.getSimpleName(); + } + + @Override + public ConfigKey[] getConfigKeys() { + return new ConfigKey[] {MaxNumberOfVCPUSPerVM}; + } } diff --git a/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java b/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java index a090b1106ac..b4e5cbc2467 100644 --- a/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java +++ b/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java @@ -1326,6 +1326,8 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe vmr.actionsAfterCrash = Types.OnCrashBehaviour.DESTROY; vmr.actionsAfterShutdown = Types.OnNormalExit.DESTROY; vmr.otherConfig.put("vm_uuid", vmSpec.getUuid()); + vmr.VCPUsMax = (long) vmSpec.getCpus(); // FIX ME: In case of dynamic scaling this VCPU max should be the minumum of + // recommended value for that template and capacity remaining on host if (isDmcEnabled(conn, host) && vmSpec.isEnableDynamicallyScaleVm()) { //scaling is allowed @@ -1334,13 +1336,10 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe vmr.memoryDynamicMin = vmSpec.getMinRam(); vmr.memoryDynamicMax = vmSpec.getMaxRam(); if (guestOsTypeName.toLowerCase().contains("windows")) { - vmr.VCPUsMax = (long)vmSpec.getCpus(); + vmr.VCPUsMax = (long) vmSpec.getCpus(); } else { - // XenServer has a documented limit of 16 vcpus per vm - vmr.VCPUsMax = 2L * vmSpec.getCpus(); - if (vmr.VCPUsMax > 16) - { - vmr.VCPUsMax = 16L; + if (vmSpec.getVcpuMaxLimit() != null) { + vmr.VCPUsMax = (long) vmSpec.getVcpuMaxLimit(); } } } else { @@ -1352,11 +1351,11 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe vmr.memoryStaticMax = vmSpec.getMaxRam(); vmr.memoryDynamicMin = vmSpec.getMaxRam();; vmr.memoryDynamicMax = vmSpec.getMaxRam(); - vmr.VCPUsMax = (long)vmSpec.getCpus(); + + vmr.VCPUsMax = (long) vmSpec.getCpus(); } - - vmr.VCPUsAtStartup = (long)vmSpec.getCpus(); + vmr.VCPUsAtStartup = (long) vmSpec.getCpus(); vmr.consoles.clear(); VM vm = VM.create(conn, vmr); From 2c515395d771592d7a8c95f89bba4ba0783b0f1f Mon Sep 17 00:00:00 2001 From: SrikanteswaraRao Talluri Date: Fri, 16 Jan 2015 18:39:17 +0530 Subject: [PATCH 053/391] CLOUDSTACK-8161: mark the data volume related operations on LXC as skipped if RBD storage pool is not available Signed-off-by: SrikanteswaraRao Talluri --- .../component/test_storage_motion.py | 6 +++++ test/integration/smoke/test_volumes.py | 24 +++++++++++++++++-- tools/marvin/marvin/lib/common.py | 19 +++++++++++++++ 3 files changed, 47 insertions(+), 2 deletions(-) diff --git a/test/integration/component/test_storage_motion.py b/test/integration/component/test_storage_motion.py index acbf92df457..163b4472470 100644 --- a/test/integration/component/test_storage_motion.py +++ b/test/integration/component/test_storage_motion.py @@ -97,6 +97,7 @@ class TestStorageMotion(cloudstackTestCase): domain = get_domain(cls.api_client) cls.zone = get_zone(cls.api_client, cls.testClient.getZoneForTests()) cls.services['mode'] = cls.zone.networktype + cls.hypervisor = cls.testClient.getHypervisorInfo() template = get_template( cls.api_client, @@ -160,6 +161,8 @@ class TestStorageMotion(cloudstackTestCase): # 3. listVM command should return this VM.State of this VM # should be "Running" and the host should be the host # to which the VM was migrated to in a different cluster + if self.hypervisor.lower() in ["lxc"]: + self.skipTest("Migration across clusters is not supported on LXC") hosts = Host.listForMigration( self.apiclient, @@ -179,6 +182,7 @@ class TestStorageMotion(cloudstackTestCase): self.skipTest("No valid hosts for storage motion. Skipping") + host = hosts[0] self.debug("Migrating VM-ID: %s to Host: %s" % ( self.virtual_machine.id, @@ -238,6 +242,8 @@ class TestStorageMotion(cloudstackTestCase): # 3. Migrate volume of the vm to another pool. # 4. Check volume is present in the new pool and is in Ready state. + # TODO: add test case for data volume migrate + list_volumes_response = list_volumes( self.apiclient, virtualmachineid=self.virtual_machine.id, diff --git a/test/integration/smoke/test_volumes.py b/test/integration/smoke/test_volumes.py index a63879967e3..3362396b981 100644 --- a/test/integration/smoke/test_volumes.py +++ b/test/integration/smoke/test_volumes.py @@ -17,7 +17,7 @@ """ BVT tests for Volumes """ #Import Local Modules -from marvin.cloudstackTestCase import cloudstackTestCase +from marvin.cloudstackTestCase import cloudstackTestCase, unittest #from marvin.cloudstackException import * from marvin.cloudstackAPI import (deleteVolume, extractVolume, @@ -34,7 +34,8 @@ from marvin.lib.base import (ServiceOffering, StoragePool,) from marvin.lib.common import (get_domain, get_zone, - get_template) + get_template, + find_storage_pool_type) from marvin.lib.utils import checkVolumeSize from marvin.codes import SUCCESS, FAILED, XEN_SERVER from nose.plugins.attrib import attr @@ -56,7 +57,12 @@ class TestCreateVolume(cloudstackTestCase): # Get Zone, Domain and templates cls.domain = get_domain(cls.apiclient) cls.zone = get_zone(cls.apiclient, testClient.getZoneForTests()) + cls.hypervisor = testClient.getHypervisorInfo() cls.services['mode'] = cls.zone.networktype + #for LXC if the storage pool of type 'rbd' ex: ceph is not available, skip the test + if cls.hypervisor.lower() == 'lxc': + if not find_storage_pool_type(cls.apiclient, storagetype='rbd'): + raise unittest.SkipTest("RBD storage type is required for data volumes for LXC") cls.disk_offering = DiskOffering.create( cls.apiclient, cls.services["disk_offering"] @@ -263,6 +269,11 @@ class TestVolumes(cloudstackTestCase): cls.domain = get_domain(cls.apiclient) cls.zone = get_zone(cls.apiclient, testClient.getZoneForTests()) cls.services['mode'] = cls.zone.networktype + cls.hypervisor = testClient.getHypervisorInfo() + #for LXC if the storage pool of type 'rbd' ex: ceph is not available, skip the test + if cls.hypervisor.lower() == 'lxc': + if not find_storage_pool_type(cls.apiclient, storagetype='rbd'): + raise unittest.SkipTest("RBD storage type is required for data volumes for LXC") cls.disk_offering = DiskOffering.create( cls.apiclient, cls.services["disk_offering"] @@ -310,7 +321,16 @@ class TestVolumes(cloudstackTestCase): serviceofferingid=cls.service_offering.id, mode=cls.services["mode"] ) + pools = StoragePool.list(cls.apiclient) + # cls.assertEqual( + # validateList(pools)[0], + # PASS, + # "storage pool list validation failed") + + + if cls.hypervisor.lower() == 'lxc' and cls.storage_pools.type.lower() != 'rbd': + raise unittest.SkipTest("Snapshots not supported on Hyper-V or LXC") cls.volume = Volume.create( cls.apiclient, cls.services, diff --git a/tools/marvin/marvin/lib/common.py b/tools/marvin/marvin/lib/common.py index 9e3d40cdbc8..28ec02443d6 100644 --- a/tools/marvin/marvin/lib/common.py +++ b/tools/marvin/marvin/lib/common.py @@ -204,6 +204,25 @@ def get_domain(apiclient, domain_id=None, domain_name=None): return cmd_out[0] +def find_storage_pool_type(apiclient, storagetype='NetworkFileSystem'): + """ + @name : find_storage_pool_type + @Desc : Returns true if the given storage pool type exists + @Input : type : type of the storage pool[NFS, RBD, etc.,] + @Output : True : if the type of storage is found + False : if the type of storage is not found + FAILED In case the cmd failed + """ + cmd = listStoragePools.listStoragePoolsCmd() + cmd_out = apiclient.listStoragePools(cmd) + if validateList(cmd_out)[0] != PASS: + return FAILED + for storage_pool in cmd_out: + if storage_pool.type.lower() == storagetype: + return True + return False + + def get_zone(apiclient, zone_name=None, zone_id=None): ''' @name : get_zone From 065c556cf5fe92dc6a4063538280e728c2719e51 Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Sun, 18 Jan 2015 17:06:39 +0530 Subject: [PATCH 054/391] CLOUDSTACK-8166: add boundary checks in various usage parsers - Add boundary condition to continue looping if creation data is after end date - Add null pointer fix for create snapshot bug Signed-off-by: Rohit Yadav --- usage/src/com/cloud/usage/UsageManagerImpl.java | 2 +- usage/src/com/cloud/usage/parser/IPAddressUsageParser.java | 5 +++++ .../src/com/cloud/usage/parser/LoadBalancerUsageParser.java | 4 ++++ .../com/cloud/usage/parser/NetworkOfferingUsageParser.java | 4 ++++ .../com/cloud/usage/parser/PortForwardingUsageParser.java | 4 ++++ .../src/com/cloud/usage/parser/SecurityGroupUsageParser.java | 4 ++++ usage/src/com/cloud/usage/parser/StorageUsageParser.java | 4 ++++ usage/src/com/cloud/usage/parser/VMInstanceUsageParser.java | 4 ++++ usage/src/com/cloud/usage/parser/VPNUserUsageParser.java | 4 ++++ usage/src/com/cloud/usage/parser/VolumeUsageParser.java | 4 ++++ 10 files changed, 38 insertions(+), 1 deletion(-) diff --git a/usage/src/com/cloud/usage/UsageManagerImpl.java b/usage/src/com/cloud/usage/UsageManagerImpl.java index 8429c6073de..8f13aa25927 100644 --- a/usage/src/com/cloud/usage/UsageManagerImpl.java +++ b/usage/src/com/cloud/usage/UsageManagerImpl.java @@ -1726,7 +1726,7 @@ public class UsageManagerImpl extends ManagerBase implements UsageManager, Runna Long offeringId = event.getOfferingId(); Long zoneId = event.getZoneId(); Long accountId = event.getAccountId(); - long size = event.getSize(); + long size = event.getSize() == null ? 0L : event.getSize().longValue(); Date created = event.getCreateDate(); Account acct = _accountDao.findByIdIncludingRemoved(event.getAccountId()); Long domainId = acct.getDomainId(); diff --git a/usage/src/com/cloud/usage/parser/IPAddressUsageParser.java b/usage/src/com/cloud/usage/parser/IPAddressUsageParser.java index adeff01dae4..6bd35d3ca7c 100644 --- a/usage/src/com/cloud/usage/parser/IPAddressUsageParser.java +++ b/usage/src/com/cloud/usage/parser/IPAddressUsageParser.java @@ -101,6 +101,11 @@ public class IPAddressUsageParser { IpAssignDate = startDate; } + // ignore if assign date is after end date + if (IpAssignDate.after(endDate)) { + continue; + } + long currentDuration = (IpReleaseDeleteDate.getTime() - IpAssignDate.getTime()) + 1; // make sure this is an inclusive check for milliseconds (i.e. use n - m + 1 to find total number of millis to charge) updateIpUsageData(usageMap, key, usageIp.getId(), currentDuration); diff --git a/usage/src/com/cloud/usage/parser/LoadBalancerUsageParser.java b/usage/src/com/cloud/usage/parser/LoadBalancerUsageParser.java index 1b813b7a845..fd51bf14ff1 100644 --- a/usage/src/com/cloud/usage/parser/LoadBalancerUsageParser.java +++ b/usage/src/com/cloud/usage/parser/LoadBalancerUsageParser.java @@ -98,6 +98,10 @@ public class LoadBalancerUsageParser { lbCreateDate = startDate; } + if (lbCreateDate.after(endDate)) { + continue; + } + long currentDuration = (lbDeleteDate.getTime() - lbCreateDate.getTime()) + 1; // make sure this is an inclusive check for milliseconds (i.e. use n - m + 1 to find total number of millis to charge) updateLBUsageData(usageMap, key, usageLB.getId(), currentDuration); diff --git a/usage/src/com/cloud/usage/parser/NetworkOfferingUsageParser.java b/usage/src/com/cloud/usage/parser/NetworkOfferingUsageParser.java index 4a5a1d2e58c..d541ddab821 100644 --- a/usage/src/com/cloud/usage/parser/NetworkOfferingUsageParser.java +++ b/usage/src/com/cloud/usage/parser/NetworkOfferingUsageParser.java @@ -99,6 +99,10 @@ public class NetworkOfferingUsageParser { noCreateDate = startDate; } + if (noCreateDate.after(endDate)) { + continue; + } + long currentDuration = (noDeleteDate.getTime() - noCreateDate.getTime()) + 1; // make sure this is an inclusive check for milliseconds (i.e. use n - m + 1 to find total number of millis to charge) updateNOUsageData(usageMap, key, usageNO.getVmInstanceId(), currentDuration); diff --git a/usage/src/com/cloud/usage/parser/PortForwardingUsageParser.java b/usage/src/com/cloud/usage/parser/PortForwardingUsageParser.java index 37c7751cff4..f58ca0c958e 100644 --- a/usage/src/com/cloud/usage/parser/PortForwardingUsageParser.java +++ b/usage/src/com/cloud/usage/parser/PortForwardingUsageParser.java @@ -98,6 +98,10 @@ public class PortForwardingUsageParser { pfCreateDate = startDate; } + if (pfCreateDate.after(endDate)) { + continue; + } + long currentDuration = (pfDeleteDate.getTime() - pfCreateDate.getTime()) + 1; // make sure this is an inclusive check for milliseconds (i.e. use n - m + 1 to find total number of millis to charge) updatePFUsageData(usageMap, key, usagePF.getId(), currentDuration); diff --git a/usage/src/com/cloud/usage/parser/SecurityGroupUsageParser.java b/usage/src/com/cloud/usage/parser/SecurityGroupUsageParser.java index ea86d49f14f..4fce2f2a0a2 100644 --- a/usage/src/com/cloud/usage/parser/SecurityGroupUsageParser.java +++ b/usage/src/com/cloud/usage/parser/SecurityGroupUsageParser.java @@ -99,6 +99,10 @@ public class SecurityGroupUsageParser { sgCreateDate = startDate; } + if (sgCreateDate.after(endDate)) { + continue; + } + long currentDuration = (sgDeleteDate.getTime() - sgCreateDate.getTime()) + 1; // make sure this is an inclusive check for milliseconds (i.e. use n - m + 1 to find total number of millis to charge) updateSGUsageData(usageMap, key, usageSG.getVmInstanceId(), currentDuration); diff --git a/usage/src/com/cloud/usage/parser/StorageUsageParser.java b/usage/src/com/cloud/usage/parser/StorageUsageParser.java index f1091a17643..6745afe2605 100644 --- a/usage/src/com/cloud/usage/parser/StorageUsageParser.java +++ b/usage/src/com/cloud/usage/parser/StorageUsageParser.java @@ -107,6 +107,10 @@ public class StorageUsageParser { storageCreateDate = startDate; } + if (storageCreateDate.after(endDate)) { + continue; + } + long currentDuration = (storageDeleteDate.getTime() - storageCreateDate.getTime()) + 1; // make sure this is an inclusive check for milliseconds (i.e. use n - m + 1 to find total number of millis to charge) updateStorageUsageData(usageMap, key, usageStorage.getId(), currentDuration); diff --git a/usage/src/com/cloud/usage/parser/VMInstanceUsageParser.java b/usage/src/com/cloud/usage/parser/VMInstanceUsageParser.java index 684412f0458..7711bbb86c8 100644 --- a/usage/src/com/cloud/usage/parser/VMInstanceUsageParser.java +++ b/usage/src/com/cloud/usage/parser/VMInstanceUsageParser.java @@ -104,6 +104,10 @@ public class VMInstanceUsageParser { vmStartDate = startDate; } + if (vmStartDate.after(endDate)) { + continue; + } + long currentDuration = (vmEndDate.getTime() - vmStartDate.getTime()) + 1; // make sure this is an inclusive check for milliseconds (i.e. use n - m + 1 to find total number of millis to charge) switch (usageType) { diff --git a/usage/src/com/cloud/usage/parser/VPNUserUsageParser.java b/usage/src/com/cloud/usage/parser/VPNUserUsageParser.java index 619c8619398..c8ad3272df7 100644 --- a/usage/src/com/cloud/usage/parser/VPNUserUsageParser.java +++ b/usage/src/com/cloud/usage/parser/VPNUserUsageParser.java @@ -94,6 +94,10 @@ public class VPNUserUsageParser { vuCreateDate = startDate; } + if (vuCreateDate.after(endDate)) { + continue; + } + long currentDuration = (vuDeleteDate.getTime() - vuCreateDate.getTime()) + 1; // make sure this is an inclusive check for milliseconds (i.e. use n - m + 1 to find total number of millis to charge) updateVUUsageData(usageMap, key, usageVU.getUserId(), currentDuration); diff --git a/usage/src/com/cloud/usage/parser/VolumeUsageParser.java b/usage/src/com/cloud/usage/parser/VolumeUsageParser.java index 81ebdc0e515..876cfc7eb30 100644 --- a/usage/src/com/cloud/usage/parser/VolumeUsageParser.java +++ b/usage/src/com/cloud/usage/parser/VolumeUsageParser.java @@ -103,6 +103,10 @@ public class VolumeUsageParser { volCreateDate = startDate; } + if (volCreateDate.after(endDate)) { + continue; + } + long currentDuration = (volDeleteDate.getTime() - volCreateDate.getTime()) + 1; // make sure this is an inclusive check for milliseconds (i.e. use n - m + 1 to find total number of millis to charge) updateVolUsageData(usageMap, key, usageVol.getId(), currentDuration); From c06ca09b54b053b37db9d4fc47226558bf15c6ba Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Sun, 18 Jan 2015 17:20:09 +0530 Subject: [PATCH 055/391] engine/storage: add NPE check, break if parent snapshot is null Signed-off-by: Rohit Yadav --- .../apache/cloudstack/storage/helper/VMSnapshotHelperImpl.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/engine/storage/src/org/apache/cloudstack/storage/helper/VMSnapshotHelperImpl.java b/engine/storage/src/org/apache/cloudstack/storage/helper/VMSnapshotHelperImpl.java index e0fc90cc138..a1bcd45d3a2 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/helper/VMSnapshotHelperImpl.java +++ b/engine/storage/src/org/apache/cloudstack/storage/helper/VMSnapshotHelperImpl.java @@ -138,6 +138,9 @@ public class VMSnapshotHelperImpl implements VMSnapshotHelper { VMSnapshotVO current = snapshot; while (current.getParent() != null) { VMSnapshotVO parent = snapshotMap.get(current.getParent()); + if (parent == null) { + break; + } currentTO.setParent(convert2VMSnapshotTO(parent)); current = snapshotMap.get(current.getParent()); currentTO = currentTO.getParent(); From 8676ff26e09162dacb7485545354e847ca27e3fe Mon Sep 17 00:00:00 2001 From: Sanjay Tripathi Date: Wed, 17 Dec 2014 15:51:47 +0530 Subject: [PATCH 056/391] CLOUDSTACK-8066: There is not way to know the size of the snapshot created. (cherry picked from commit 9153b8bede3350750cb7a78891e0479cc72aa2ec) Signed-off-by: Rohit Yadav --- api/src/org/apache/cloudstack/api/ApiConstants.java | 1 + .../cloudstack/api/response/SnapshotResponse.java | 8 ++++++++ .../engine/subsystem/api/storage/SnapshotInfo.java | 2 ++ .../cloudstack/storage/snapshot/SnapshotObject.java | 10 ++++++++++ server/src/com/cloud/api/ApiResponseHelper.java | 1 + 5 files changed, 22 insertions(+) diff --git a/api/src/org/apache/cloudstack/api/ApiConstants.java b/api/src/org/apache/cloudstack/api/ApiConstants.java index 762ad7097fc..6bd5ddede3e 100755 --- a/api/src/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/org/apache/cloudstack/api/ApiConstants.java @@ -613,6 +613,7 @@ public class ApiConstants { public static final String REGION_LEVEL_VPC = "regionlevelvpc"; public static final String STRECHED_L2_SUBNET = "strechedl2subnet"; public static final String NETWORK_SPANNED_ZONES = "zonesnetworkspans"; + public static final String PHYSICAL_SIZE = "physicalsize"; public enum HostDetails { all, capacity, events, stats, min; diff --git a/api/src/org/apache/cloudstack/api/response/SnapshotResponse.java b/api/src/org/apache/cloudstack/api/response/SnapshotResponse.java index 10cb202ba93..aa8de5c6fd4 100644 --- a/api/src/org/apache/cloudstack/api/response/SnapshotResponse.java +++ b/api/src/org/apache/cloudstack/api/response/SnapshotResponse.java @@ -86,6 +86,10 @@ public class SnapshotResponse extends BaseResponse implements ControlledEntityRe @Param(description = "the state of the snapshot. BackedUp means that snapshot is ready to be used; Creating - the snapshot is being allocated on the primary storage; BackingUp - the snapshot is being backed up on secondary storage") private Snapshot.State state; + @SerializedName(ApiConstants.PHYSICAL_SIZE) + @Param(description = "physical size of backedup snapshot on image store") + private long physicalSize; + @SerializedName(ApiConstants.ZONE_ID) @Param(description = "id of the availability zone") private String zoneId; @@ -166,6 +170,10 @@ public class SnapshotResponse extends BaseResponse implements ControlledEntityRe this.state = state; } + public void setPhysicaSize(long physicalSize) { + this.physicalSize = physicalSize; + } + @Override public void setProjectId(String projectId) { this.projectId = projectId; diff --git a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotInfo.java b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotInfo.java index 7865965fa93..0f05ca186f3 100644 --- a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotInfo.java +++ b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotInfo.java @@ -36,4 +36,6 @@ public interface SnapshotInfo extends DataObject, Snapshot { ObjectInDataStoreStateMachine.State getStatus(); boolean isRevertable(); + + long getPhysicalSize(); } diff --git a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotObject.java b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotObject.java index c64f78fd01c..60ff31e4342 100644 --- a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotObject.java +++ b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotObject.java @@ -138,6 +138,16 @@ public class SnapshotObject implements SnapshotInfo { return false; } + @Override + public long getPhysicalSize() { + long physicalSize = 0; + SnapshotDataStoreVO snapshotStore = snapshotStoreDao.findBySnapshot(snapshot.getId(), DataStoreRole.Image); + if (snapshotStore != null) { + physicalSize = snapshotStore.getPhysicalSize(); + } + return physicalSize; + } + @Override public VolumeInfo getBaseVolume() { return volFactory.getVolume(snapshot.getVolumeId()); diff --git a/server/src/com/cloud/api/ApiResponseHelper.java b/server/src/com/cloud/api/ApiResponseHelper.java index f4d02936a31..b8d4b3f541b 100755 --- a/server/src/com/cloud/api/ApiResponseHelper.java +++ b/server/src/com/cloud/api/ApiResponseHelper.java @@ -470,6 +470,7 @@ public class ApiResponseHelper implements ResponseGenerator { snapshotResponse.setRevertable(false); } else { snapshotResponse.setRevertable(snapshotInfo.isRevertable()); + snapshotResponse.setPhysicaSize(snapshotInfo.getPhysicalSize()); } // set tag information From a1791cb4a8fb2cfd455848ca47585d94d9a309e7 Mon Sep 17 00:00:00 2001 From: Saksham Srivastava Date: Thu, 18 Dec 2014 14:26:33 +0530 Subject: [PATCH 057/391] CLOUDSTACK-8088: VM scale up is failing in vmware with Unable to execute ScaleVmCommand due to java.lang.NullPointerException (cherry picked from commit 1df0453d27e8378065c15878067fc9d2dc961e30) Signed-off-by: Rohit Yadav --- api/src/com/cloud/hypervisor/HypervisorGuru.java | 3 +++ .../src/com/cloud/vm/VirtualMachineManagerImpl.java | 7 +++++++ .../src/com/cloud/baremetal/manager/BareMetalGuru.java | 6 ++++++ .../src/com/cloud/hypervisor/hyperv/guru/HypervGuru.java | 7 +++++++ .../ovm/src/com/cloud/ovm/hypervisor/OvmGuru.java | 8 ++++++++ .../simulator/src/com/cloud/simulator/SimulatorGuru.java | 8 ++++++++ .../vmware/src/com/cloud/hypervisor/guru/VMwareGuru.java | 9 +++++++++ .../src/com/cloud/hypervisor/XenServerGuru.java | 6 ++++++ server/src/com/cloud/hypervisor/HypervisorGuruBase.java | 6 ++++++ server/src/com/cloud/hypervisor/KVMGuru.java | 8 ++++++++ server/src/com/cloud/hypervisor/LXCGuru.java | 7 +++++++ 11 files changed, 75 insertions(+) diff --git a/api/src/com/cloud/hypervisor/HypervisorGuru.java b/api/src/com/cloud/hypervisor/HypervisorGuru.java index 583e082f860..6a09dd257af 100644 --- a/api/src/com/cloud/hypervisor/HypervisorGuru.java +++ b/api/src/com/cloud/hypervisor/HypervisorGuru.java @@ -17,6 +17,7 @@ package com.cloud.hypervisor; import java.util.List; +import java.util.Map; import org.apache.cloudstack.framework.config.ConfigKey; @@ -81,4 +82,6 @@ public interface HypervisorGuru extends Adapter { List finalizeExpungeNics(VirtualMachine vm, List nics); List finalizeExpungeVolumes(VirtualMachine vm); + + Map getClusterSettings(long vmId); } diff --git a/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java index ab5f6a3f046..038c7afe052 100755 --- a/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java @@ -3487,6 +3487,13 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac newServiceOffering.getSpeed(), minMemory * 1024L * 1024L, newServiceOffering.getRamSize() * 1024L * 1024L, newServiceOffering.getLimitCpuUse()); Long dstHostId = vm.getHostId(); + if(vm.getHypervisorType().equals(HypervisorType.VMware)) { + HypervisorGuru hvGuru = _hvGuruMgr.getGuru(vm.getHypervisorType()); + Map details = null; + details = hvGuru.getClusterSettings(vm.getId()); + reconfigureCmd.getVirtualMachine().setDetails(details); + } + ItWorkVO work = new ItWorkVO(UUID.randomUUID().toString(), _nodeId, State.Running, vm.getType(), vm.getId()); work.setStep(Step.Prepare); work.setResourceType(ItWorkVO.ResourceType.Host); diff --git a/plugins/hypervisors/baremetal/src/com/cloud/baremetal/manager/BareMetalGuru.java b/plugins/hypervisors/baremetal/src/com/cloud/baremetal/manager/BareMetalGuru.java index 57d439bcd6c..8f439cdc1dd 100755 --- a/plugins/hypervisors/baremetal/src/com/cloud/baremetal/manager/BareMetalGuru.java +++ b/plugins/hypervisors/baremetal/src/com/cloud/baremetal/manager/BareMetalGuru.java @@ -84,4 +84,10 @@ public class BareMetalGuru extends HypervisorGuruBase implements HypervisorGuru public boolean trackVmHostChange() { return false; } + + @Override + public Map getClusterSettings(long vmId) { + return null; + } + } diff --git a/plugins/hypervisors/hyperv/src/com/cloud/hypervisor/hyperv/guru/HypervGuru.java b/plugins/hypervisors/hyperv/src/com/cloud/hypervisor/hyperv/guru/HypervGuru.java index 4e8d3d5524c..2a631429156 100644 --- a/plugins/hypervisors/hyperv/src/com/cloud/hypervisor/hyperv/guru/HypervGuru.java +++ b/plugins/hypervisors/hyperv/src/com/cloud/hypervisor/hyperv/guru/HypervGuru.java @@ -20,6 +20,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.List; +import java.util.Map; import javax.ejb.Local; import javax.inject.Inject; @@ -194,4 +195,10 @@ public class HypervGuru extends HypervisorGuruBase implements HypervisorGuru { public final boolean trackVmHostChange() { return false; } + + @Override + public Map getClusterSettings(long vmId) { + return null; + } + } diff --git a/plugins/hypervisors/ovm/src/com/cloud/ovm/hypervisor/OvmGuru.java b/plugins/hypervisors/ovm/src/com/cloud/ovm/hypervisor/OvmGuru.java index 236078f526e..90e4b5742b6 100755 --- a/plugins/hypervisors/ovm/src/com/cloud/ovm/hypervisor/OvmGuru.java +++ b/plugins/hypervisors/ovm/src/com/cloud/ovm/hypervisor/OvmGuru.java @@ -16,6 +16,8 @@ // under the License. package com.cloud.ovm.hypervisor; +import java.util.Map; + import javax.ejb.Local; import javax.inject.Inject; @@ -58,4 +60,10 @@ public class OvmGuru extends HypervisorGuruBase implements HypervisorGuru { return true; } + @Override + public Map getClusterSettings(long vmId) { + return null; + } + + } diff --git a/plugins/hypervisors/simulator/src/com/cloud/simulator/SimulatorGuru.java b/plugins/hypervisors/simulator/src/com/cloud/simulator/SimulatorGuru.java index 5b9eab4d3b6..378134a9856 100644 --- a/plugins/hypervisors/simulator/src/com/cloud/simulator/SimulatorGuru.java +++ b/plugins/hypervisors/simulator/src/com/cloud/simulator/SimulatorGuru.java @@ -16,6 +16,8 @@ // under the License. package com.cloud.simulator; +import java.util.Map; + import javax.ejb.Local; import javax.inject.Inject; @@ -56,4 +58,10 @@ public class SimulatorGuru extends HypervisorGuruBase implements HypervisorGuru public boolean trackVmHostChange() { return false; } + + @Override + public Map getClusterSettings(long vmId) { + return null; + } + } diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/guru/VMwareGuru.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/guru/VMwareGuru.java index c0711f211b6..808cb5bdeea 100644 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/guru/VMwareGuru.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/guru/VMwareGuru.java @@ -552,4 +552,13 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co return commands; } + + @Override + public Map getClusterSettings(long vmId) { + Map details = new HashMap(); + long clusterId = getClusterId(vmId); + details.put(VmwareReserveCpu.key(), VmwareReserveCpu.valueIn(clusterId).toString()); + details.put(VmwareReserveMemory.key(), VmwareReserveMemory.valueIn(clusterId).toString()); + return details; + } } diff --git a/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/XenServerGuru.java b/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/XenServerGuru.java index b17048b435c..f8aabff64a0 100644 --- a/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/XenServerGuru.java +++ b/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/XenServerGuru.java @@ -18,6 +18,7 @@ package com.cloud.hypervisor; import java.util.ArrayList; import java.util.List; +import java.util.Map; import javax.ejb.Local; import javax.inject.Inject; @@ -128,6 +129,11 @@ public class XenServerGuru extends HypervisorGuruBase implements HypervisorGuru, return true; } + @Override + public Map getClusterSettings(long vmId) { + return null; + } + @Override public List finalizeExpungeVolumes(VirtualMachine vm) { List commands = new ArrayList(); diff --git a/server/src/com/cloud/hypervisor/HypervisorGuruBase.java b/server/src/com/cloud/hypervisor/HypervisorGuruBase.java index 8ffd6ea93f2..308c327177f 100644 --- a/server/src/com/cloud/hypervisor/HypervisorGuruBase.java +++ b/server/src/com/cloud/hypervisor/HypervisorGuruBase.java @@ -190,4 +190,10 @@ public abstract class HypervisorGuruBase extends AdapterBase implements Hypervis public List finalizeExpungeVolumes(VirtualMachine vm) { return null; } + + @Override + public Map getClusterSettings(long vmId) { + return null; + } + } diff --git a/server/src/com/cloud/hypervisor/KVMGuru.java b/server/src/com/cloud/hypervisor/KVMGuru.java index b6a765154c5..a2534bdef2a 100644 --- a/server/src/com/cloud/hypervisor/KVMGuru.java +++ b/server/src/com/cloud/hypervisor/KVMGuru.java @@ -16,6 +16,8 @@ // under the License. package com.cloud.hypervisor; +import java.util.Map; + import javax.ejb.Local; import javax.inject.Inject; @@ -86,4 +88,10 @@ public class KVMGuru extends HypervisorGuruBase implements HypervisorGuru { public boolean trackVmHostChange() { return false; } + + @Override + public Map getClusterSettings(long vmId) { + return null; + } + } diff --git a/server/src/com/cloud/hypervisor/LXCGuru.java b/server/src/com/cloud/hypervisor/LXCGuru.java index 025c498cd4a..2becdbd79e6 100644 --- a/server/src/com/cloud/hypervisor/LXCGuru.java +++ b/server/src/com/cloud/hypervisor/LXCGuru.java @@ -16,6 +16,8 @@ // under the License. package com.cloud.hypervisor; +import java.util.Map; + import javax.ejb.Local; import javax.inject.Inject; @@ -73,4 +75,9 @@ public class LXCGuru extends HypervisorGuruBase implements HypervisorGuru { public boolean trackVmHostChange() { return false; } + + @Override + public Map getClusterSettings(long vmId) { + return null; + } } From 5f117e6be9c098a41e104ed0be1ecf1dc0e964c9 Mon Sep 17 00:00:00 2001 From: Rajesh Battala Date: Wed, 18 Jun 2014 15:42:23 +0530 Subject: [PATCH 058/391] CLOUDSTACK-6920 Support listing of LBHealthcheck policy with LBHealthcheck policy ID (cherry picked from commit c7b23d0a10fa6fc55820f298cce658bff0b8125c) Signed-off-by: Rohit Yadav --- .../network/lb/LoadBalancingRulesService.java | 2 ++ .../ListLBHealthCheckPoliciesCmd.java | 21 +++++++++++++++++-- .../lb/LoadBalancingRulesManagerImpl.java | 16 ++++++++++++-- 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/api/src/com/cloud/network/lb/LoadBalancingRulesService.java b/api/src/com/cloud/network/lb/LoadBalancingRulesService.java index 3e1101453c2..50b39d2f338 100644 --- a/api/src/com/cloud/network/lb/LoadBalancingRulesService.java +++ b/api/src/com/cloud/network/lb/LoadBalancingRulesService.java @@ -161,4 +161,6 @@ public interface LoadBalancingRulesService { HealthCheckPolicy updateLBHealthCheckPolicy(long id, String customId, Boolean forDisplay); LoadBalancer findLbByStickinessId(long stickinessPolicyId); + + Long findLBIdByHealtCheckPolicyId(long lbHealthCheckPolicy); } diff --git a/api/src/org/apache/cloudstack/api/command/user/loadbalancer/ListLBHealthCheckPoliciesCmd.java b/api/src/org/apache/cloudstack/api/command/user/loadbalancer/ListLBHealthCheckPoliciesCmd.java index 7f78da64097..3f2082af5ca 100644 --- a/api/src/org/apache/cloudstack/api/command/user/loadbalancer/ListLBHealthCheckPoliciesCmd.java +++ b/api/src/org/apache/cloudstack/api/command/user/loadbalancer/ListLBHealthCheckPoliciesCmd.java @@ -29,6 +29,8 @@ import org.apache.cloudstack.api.response.LBHealthCheckResponse; import org.apache.cloudstack.api.response.ListResponse; import org.apache.log4j.Logger; + +import com.cloud.exception.InvalidParameterValueException; import com.cloud.network.rules.HealthCheckPolicy; import com.cloud.network.rules.LoadBalancer; @@ -45,13 +47,15 @@ public class ListLBHealthCheckPoliciesCmd extends BaseListCmd { @Parameter(name = ApiConstants.LBID, type = CommandType.UUID, entityType = FirewallRuleResponse.class, - required = true, description = "the ID of the load balancer rule") private Long lbRuleId; @Parameter(name = ApiConstants.FOR_DISPLAY, type = CommandType.BOOLEAN, description = "list resources by display flag; only ROOT admin is eligible to pass this parameter", since = "4.4", authorized = {RoleType.Admin}) private Boolean display; + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = LBHealthCheckResponse.class, description = "the ID of the healthcheck policy", since = "4.4") + private Long id; + // /////////////////////////////////////////////////// // ///////////////// Accessors /////////////////////// // /////////////////////////////////////////////////// @@ -59,6 +63,10 @@ public class ListLBHealthCheckPoliciesCmd extends BaseListCmd { return lbRuleId; } + public Long getId() { + return id; + } + public boolean getDisplay() { if (display != null) { return display; @@ -78,9 +86,18 @@ public class ListLBHealthCheckPoliciesCmd extends BaseListCmd { @Override public void execute() { List hcpResponses = new ArrayList(); - LoadBalancer lb = _lbService.findById(getLbRuleId()); ListResponse response = new ListResponse(); + Long lbRuleId = getLbRuleId(); + Long hId = getId(); + if(lbRuleId == null) { + if(hId != null) { + lbRuleId = _lbService.findLBIdByHealtCheckPolicyId(hId); + } else { + throw new InvalidParameterValueException("Either LB Ruleid or HealthCheckpolicy Id should be specified"); + } + } + LoadBalancer lb = _lbService.findById(lbRuleId); if (lb != null) { List healthCheckPolicies = _lbService.searchForLBHealthCheckPolicies(this); LBHealthCheckResponse spResponse = _responseGenerator.createLBHealthCheckPolicyResponse(healthCheckPolicies, lb); diff --git a/server/src/com/cloud/network/lb/LoadBalancingRulesManagerImpl.java b/server/src/com/cloud/network/lb/LoadBalancingRulesManagerImpl.java index fbb862e6938..d7a85b6ff4f 100755 --- a/server/src/com/cloud/network/lb/LoadBalancingRulesManagerImpl.java +++ b/server/src/com/cloud/network/lb/LoadBalancingRulesManagerImpl.java @@ -2263,15 +2263,18 @@ public class LoadBalancingRulesManagerImpl extends ManagerBase implements public List searchForLBHealthCheckPolicies(ListLBHealthCheckPoliciesCmd cmd) throws PermissionDeniedException { Account caller = CallContext.current().getCallingAccount(); Long loadBalancerId = cmd.getLbRuleId(); + Long policyId = cmd.getId(); boolean forDisplay = cmd.getDisplay(); - + if(loadBalancerId == null) { + loadBalancerId = findLBIdByHealtCheckPolicyId(policyId); + } LoadBalancerVO loadBalancer = _lbDao.findById(loadBalancerId); if (loadBalancer == null) { return null; } _accountMgr.checkAccess(caller, null, true, loadBalancer); - List hcDbpolicies = _lb2healthcheckDao.listByLoadBalancerIdAndDisplayFlag(cmd.getLbRuleId(), forDisplay); + List hcDbpolicies = _lb2healthcheckDao.listByLoadBalancerIdAndDisplayFlag(loadBalancerId, forDisplay); return hcDbpolicies; } @@ -2569,4 +2572,13 @@ public class LoadBalancingRulesManagerImpl extends ManagerBase implements return _lb2healthcheckDao.findById(id); } + @Override + public Long findLBIdByHealtCheckPolicyId(long lbHealthCheckPolicy) { + LBHealthCheckPolicyVO policy= _lb2healthcheckDao.findById(lbHealthCheckPolicy); + if(policy != null) { + return policy.getLoadBalancerId(); + } + return null; + } + } From f6564b0b03362093efd1766a64e34184208eec3c Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Sun, 18 Jan 2015 17:40:37 +0530 Subject: [PATCH 059/391] CLOUDSTACK-8167: publish volume uuid on event bus in create snapshot cmd Signed-off-by: Rohit Yadav --- .../api/command/user/snapshot/CreateSnapshotCmd.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/api/src/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java b/api/src/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java index df7fe8296bc..7bba6208848 100644 --- a/api/src/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java +++ b/api/src/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java @@ -100,6 +100,14 @@ public class CreateSnapshotCmd extends BaseAsyncCreateCmd { return volumeId; } + public String getVolumeUuid() { + Volume volume = (Volume)this._entityMgr.findById(Volume.class, getVolumeId()); + if (volume == null) { + throw new InvalidParameterValueException("Unable to find volume's UUID"); + } + return volume.getUuid(); + } + public Long getPolicyId() { if (policyId != null) { return policyId; @@ -159,7 +167,7 @@ public class CreateSnapshotCmd extends BaseAsyncCreateCmd { @Override public String getEventDescription() { - return "creating snapshot for volume: " + getVolumeId(); + return "creating snapshot for volume: " + getVolumeUuid(); } @Override @@ -181,7 +189,7 @@ public class CreateSnapshotCmd extends BaseAsyncCreateCmd { @Override public void execute() { s_logger.info("VOLSS: createSnapshotCmd starts:" + System.currentTimeMillis()); - CallContext.current().setEventDetails("Volume Id: " + getVolumeId()); + CallContext.current().setEventDetails("Volume Id: " + getVolumeUuid()); Snapshot snapshot; try { snapshot = From d807bf0a4eb6fd5b5e69318c519ad1eb0d1de5b9 Mon Sep 17 00:00:00 2001 From: Likitha Shetty Date: Mon, 13 Oct 2014 16:26:47 +0530 Subject: [PATCH 060/391] CLOUDSTACK-8107. Failed to create snapshot from volume when the task is performed repeatedly in zone wide primary Storage. While taking a snapshot of a volume, CS chooses the endpoint to perform backup snapshot operation by selecting any host that has the storage containing the volume mounted on it. Instead, if the volume is attached to a VM, the endpoint chosen by CS should be the host that contains the VM. (cherry picked from commit a75a43137316a60b20760aa5015d97f55520fd16) Signed-off-by: Rohit Yadav --- .../storage/endpoint/DefaultEndPointSelector.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/engine/storage/src/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java b/engine/storage/src/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java index 18c8991e0c2..7067b8c4f76 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java +++ b/engine/storage/src/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java @@ -205,13 +205,21 @@ public class DefaultEndPointSelector implements EndPointSelector { public EndPoint select(DataObject srcData, DataObject destData, StorageAction action) { if (action == StorageAction.BACKUPSNAPSHOT && srcData.getDataStore().getRole() == DataStoreRole.Primary) { SnapshotInfo srcSnapshot = (SnapshotInfo)srcData; + VolumeInfo volumeInfo = srcSnapshot.getBaseVolume(); + VirtualMachine vm = volumeInfo.getAttachedVM(); if (srcSnapshot.getHypervisorType() == Hypervisor.HypervisorType.KVM) { - VolumeInfo volumeInfo = srcSnapshot.getBaseVolume(); - VirtualMachine vm = volumeInfo.getAttachedVM(); if (vm != null && vm.getState() == VirtualMachine.State.Running) { return getEndPointFromHostId(vm.getHostId()); } } + if (srcSnapshot.getHypervisorType() == Hypervisor.HypervisorType.VMware) { + if (vm != null) { + Long hostId = vm.getHostId() != null ? vm.getHostId() : vm.getLastHostId(); + if (hostId != null) { + return getEndPointFromHostId(hostId); + } + } + } } return select(srcData, destData); } From 466e6d15a44cec696abdea7e01b60f653bd641bb Mon Sep 17 00:00:00 2001 From: Sanjay Tripathi Date: Fri, 19 Dec 2014 14:26:48 +0530 Subject: [PATCH 061/391] CLOUDSTACK-8097: Failed to create volume snapshot after vm live migration across clusters. (cherry picked from commit 0c4128e024b519f24625ad8cd6b53ddf0137728e) Signed-off-by: Rohit Yadav --- .../orchestration/VolumeOrchestrator.java | 16 +++++- .../com/cloud/storage/dao/SnapshotDao.java | 2 + .../cloud/storage/dao/SnapshotDaoImpl.java | 10 ++++ .../datastore/db/SnapshotDataStoreDao.java | 4 ++ .../snapshot/XenserverSnapshotStrategy.java | 55 ++++++++++++------- .../image/db/SnapshotDataStoreDaoImpl.java | 47 +++++++++++++++- 6 files changed, 112 insertions(+), 22 deletions(-) diff --git a/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java index 6f6a29e47b1..0895e84c551 100644 --- a/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java +++ b/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java @@ -57,6 +57,7 @@ import org.apache.cloudstack.framework.jobs.AsyncJobManager; import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO; import org.apache.cloudstack.storage.command.CommandResult; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; @@ -95,6 +96,7 @@ import com.cloud.storage.VMTemplateStorageResourceAssoc; import com.cloud.storage.Volume; import com.cloud.storage.Volume.Type; import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.SnapshotDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.storage.dao.VolumeDetailsDao; import com.cloud.template.TemplateManager; @@ -141,6 +143,10 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati @Inject protected VolumeDao _volumeDao; @Inject + protected SnapshotDao _snapshotDao; + @Inject + protected SnapshotDataStoreDao _snapshotDataStoreDao; + @Inject protected ResourceLimitService _resourceLimitMgr; @Inject VolumeDetailsDao _volDetailDao; @@ -916,8 +922,14 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati try { VolumeApiResult result = future.get(); if (result.isFailed()) { - s_logger.error("migrate volume failed:" + result.getResult()); - throw new StorageUnavailableException("migrate volume failed: " + result.getResult(), destPool.getId()); + s_logger.error("Migrate volume failed:" + result.getResult()); + throw new StorageUnavailableException("Migrate volume failed: " + result.getResult(), destPool.getId()); + } else { + // update the volumeId for snapshots on secondary + if (!_snapshotDao.listByVolumeId(vol.getId()).isEmpty()) { + _snapshotDao.updateVolumeIds(vol.getId(), result.getVolume().getId()); + _snapshotDataStoreDao.updateVolumeIds(vol.getId(), result.getVolume().getId()); + } } return result.getVolume(); } catch (InterruptedException e) { diff --git a/engine/schema/src/com/cloud/storage/dao/SnapshotDao.java b/engine/schema/src/com/cloud/storage/dao/SnapshotDao.java index f55352b2ef2..ff2e4457441 100644 --- a/engine/schema/src/com/cloud/storage/dao/SnapshotDao.java +++ b/engine/schema/src/com/cloud/storage/dao/SnapshotDao.java @@ -57,4 +57,6 @@ public interface SnapshotDao extends GenericDao, StateDao listAllByStatus(Snapshot.State... status); + void updateVolumeIds(long oldVolId, long newVolId); + } diff --git a/engine/schema/src/com/cloud/storage/dao/SnapshotDaoImpl.java b/engine/schema/src/com/cloud/storage/dao/SnapshotDaoImpl.java index 204447c6909..84a92d7bf0d 100644 --- a/engine/schema/src/com/cloud/storage/dao/SnapshotDaoImpl.java +++ b/engine/schema/src/com/cloud/storage/dao/SnapshotDaoImpl.java @@ -41,6 +41,7 @@ import com.cloud.utils.db.DB; import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.GenericSearchBuilder; +import com.cloud.utils.db.UpdateBuilder; import com.cloud.utils.db.JoinBuilder.JoinType; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; @@ -326,4 +327,13 @@ public class SnapshotDaoImpl extends GenericDaoBase implements return true; } + @Override + public void updateVolumeIds(long oldVolId, long newVolId) { + SearchCriteria sc = VolumeIdSearch.create(); + sc.setParameters("volumeId", oldVolId); + SnapshotVO snapshot = createForUpdate(); + snapshot.setVolumeId(newVolId); + UpdateBuilder ub = getUpdateBuilder(snapshot); + update(ub, sc, null); + } } diff --git a/engine/schema/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java b/engine/schema/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java index e24c0351e46..231b241408b 100644 --- a/engine/schema/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java +++ b/engine/schema/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java @@ -56,4 +56,8 @@ public interface SnapshotDataStoreDao extends GenericDao= deltaSnap) { - fullBackup = true; - } else { - fullBackup = false; + for (i = 1; i < deltaSnap; i++) { + Long prevBackupId = parentSnapshotOnBackupStore.getParentSnapshotId(); + if (prevBackupId == 0) { + break; + } + parentSnapshotOnBackupStore = snapshotStoreDao.findBySnapshot(prevBackupId, DataStoreRole.Image); + if (parentSnapshotOnBackupStore == null) { + break; + } + } + + if (i >= deltaSnap) { + fullBackup = true; + } else { + fullBackup = false; + } + } else { + // if there is an snapshot entry for previousPool(primary storage) of migrated volume, delete it becasue CS created one more snapshot entry for current pool + snapshotStoreDao.remove(oldestSnapshotOnPrimary.getId()); + } } } diff --git a/engine/storage/src/org/apache/cloudstack/storage/image/db/SnapshotDataStoreDaoImpl.java b/engine/storage/src/org/apache/cloudstack/storage/image/db/SnapshotDataStoreDaoImpl.java index 28d65981065..ea73ecd7e33 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/image/db/SnapshotDataStoreDaoImpl.java +++ b/engine/storage/src/org/apache/cloudstack/storage/image/db/SnapshotDataStoreDaoImpl.java @@ -54,6 +54,7 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase snapshotSearch; private SearchBuilder storeSnapshotSearch; private SearchBuilder snapshotIdSearch; + private SearchBuilder volumeIdSearch; private final String parentSearch = "select store_id, store_role, snapshot_id from cloud.snapshot_store_ref where store_id = ? " + " and store_role = ? and volume_id = ? and state = 'Ready'" + " order by created DESC " + " limit 1"; @@ -61,6 +62,10 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase params) throws ConfigurationException { @@ -110,6 +115,10 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase sc = volumeIdSearch.create(); + sc.setParameters("volume_id", oldVolId); + SnapshotDataStoreVO snapshot = createForUpdate(); + snapshot.setVolumeId(newVolId); + UpdateBuilder ub = getUpdateBuilder(snapshot); + update(ub, sc, null); + } } From 1e87f3b80bbdc7e9e101bdc5f4b5e05885b55491 Mon Sep 17 00:00:00 2001 From: Kishan Kavala Date: Wed, 7 Jan 2015 14:52:42 +0530 Subject: [PATCH 062/391] Bug-Id: CLOUDSTACK-3439: Include dynamically created nics in Prepare for migration command in KVM (cherry picked from commit f767adfe71dda82c45cf376ba71518e2ad84cc98) Signed-off-by: Rohit Yadav --- api/src/com/cloud/vm/NicProfile.java | 8 ++ .../service/NetworkOrchestrationService.java | 2 + .../cloud/vm/VirtualMachineManagerImpl.java | 1 + .../orchestration/NetworkOrchestrator.java | 86 +++++++++++++++++++ .../com/cloud/network/dao/IPAddressDao.java | 1 + .../cloud/network/dao/IPAddressDaoImpl.java | 7 ++ .../cloud/hypervisor/HypervisorGuruBase.java | 4 + .../com/cloud/vpc/MockNetworkManagerImpl.java | 5 ++ 8 files changed, 114 insertions(+) diff --git a/api/src/com/cloud/vm/NicProfile.java b/api/src/com/cloud/vm/NicProfile.java index 4dd7f68ee33..e9e9dc54ad1 100644 --- a/api/src/com/cloud/vm/NicProfile.java +++ b/api/src/com/cloud/vm/NicProfile.java @@ -70,6 +70,10 @@ public class NicProfile implements InternalIdentity, Serializable { return name; } + public void setName(String name) { + this.name = name; + } + public String getDns2() { return dns2; } @@ -371,4 +375,8 @@ public class NicProfile implements InternalIdentity, Serializable { this.ip6Dns2 = ip6Dns2; } + public void setNetworId(long networkId){ + this.networkId = networkId; + } + } diff --git a/engine/api/src/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java b/engine/api/src/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java index b2108ac9075..5412ee4ce1f 100755 --- a/engine/api/src/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java +++ b/engine/api/src/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java @@ -217,4 +217,6 @@ public interface NetworkOrchestrationService { void removeDhcpServiceInSubnet(Nic nic); boolean resourceCountNeedsUpdate(NetworkOffering ntwkOff, ACLType aclType); + + void prepareAllNicsForMigration(VirtualMachineProfile vm, DeployDestination dest); } diff --git a/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java index 038c7afe052..b992ed95125 100755 --- a/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java @@ -3246,6 +3246,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm); _networkMgr.prepareNicForMigration(profile, dest); + volumeMgr.prepareForMigration(profile, dest); VirtualMachineTO to = toVmTO(profile); diff --git a/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java b/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java index b28a26b846b..c84fb8d12cd 100755 --- a/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java +++ b/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java @@ -37,6 +37,7 @@ import javax.ejb.Local; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.network.Networks; import org.apache.log4j.Logger; import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.context.CallContext; @@ -1376,6 +1377,11 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra @Override public void prepareNicForMigration(VirtualMachineProfile vm, DeployDestination dest) { + if(vm.getType().equals(VirtualMachine.Type.DomainRouter) && vm.getHypervisorType().equals(HypervisorType.KVM)){ + //Include nics hot plugged and not stored in DB + prepareAllNicsForMigration(vm, dest); + return; + } List nics = _nicDao.listByVmId(vm.getId()); ReservationContext context = new ReservationContextImpl(UUID.randomUUID().toString(), null, null); for (NicVO nic : nics) { @@ -1409,6 +1415,86 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra } } + /* + Prepare All Nics for migration including the nics dynamically created and not stored in DB + This is a temporary workaround work KVM migration + Once clean fix is added by stored dynamically nics is DB, this workaround won't be needed + */ + @Override + public void prepareAllNicsForMigration(VirtualMachineProfile vm, DeployDestination dest) { + List nics = _nicDao.listByVmId(vm.getId()); + ReservationContext context = new ReservationContextImpl(UUID.randomUUID().toString(), null, null); + Long guestNetworkId = null; + for (NicVO nic : nics) { + NetworkVO network = _networksDao.findById(nic.getNetworkId()); + if(network.getTrafficType().equals(TrafficType.Guest) && network.getGuestType().equals(GuestType.Isolated)){ + guestNetworkId = network.getId(); + } + Integer networkRate = _networkModel.getNetworkRate(network.getId(), vm.getId()); + + NetworkGuru guru = AdapterBase.getAdapterByName(networkGurus, network.getGuruName()); + NicProfile profile = new NicProfile(nic, network, nic.getBroadcastUri(), nic.getIsolationUri(), networkRate, + _networkModel.isSecurityGroupSupportedInNetwork(network), _networkModel.getNetworkTag(vm.getHypervisorType(), network)); + if(guru instanceof NetworkMigrationResponder){ + if(!((NetworkMigrationResponder) guru).prepareMigration(profile, network, vm, dest, context)){ + s_logger.error("NetworkGuru "+guru+" prepareForMigration failed."); // XXX: Transaction error + } + } + List providersToImplement = getNetworkProviders(network.getId()); + for (NetworkElement element : networkElements) { + if (providersToImplement.contains(element.getProvider())) { + if (!_networkModel.isProviderEnabledInPhysicalNetwork(_networkModel.getPhysicalNetworkId(network), element.getProvider().getName())) { + throw new CloudRuntimeException("Service provider " + element.getProvider().getName() + " either doesn't exist or is not enabled in physical network id: " + network.getPhysicalNetworkId()); + } + if(element instanceof NetworkMigrationResponder){ + if(!((NetworkMigrationResponder) element).prepareMigration(profile, network, vm, dest, context)){ + s_logger.error("NetworkElement "+element+" prepareForMigration failed."); // XXX: Transaction error + } + } + } + } + guru.updateNicProfile(profile, network); + vm.addNic(profile); + } + + List addedURIs = new ArrayList(); + if(guestNetworkId != null){ + List publicIps = _ipAddressDao.listByAssociatedNetwork(guestNetworkId, null); + for (IPAddressVO userIp : publicIps){ + PublicIp publicIp = PublicIp.createFromAddrAndVlan(userIp, _vlanDao.findById(userIp.getVlanId())); + URI broadcastUri = BroadcastDomainType.Vlan.toUri(publicIp.getVlanTag()); + long ntwkId = publicIp.getNetworkId(); + Nic nic = _nicDao.findByNetworkIdInstanceIdAndBroadcastUri(ntwkId, vm.getId(), + broadcastUri.toString()); + if(nic == null && !addedURIs.contains(broadcastUri.toString())){ + //Nic details are not available in DB + //Create nic profile for migration + s_logger.debug("Creating nic profile for migration. BroadcastUri: "+broadcastUri.toString()+" NetworkId: "+ntwkId+" Vm: "+vm.getId()); + NetworkVO network = _networksDao.findById(ntwkId); + Integer networkRate = _networkModel.getNetworkRate(network.getId(), vm.getId()); + NetworkGuru guru = AdapterBase.getAdapterByName(networkGurus, network.getGuruName()); + NicProfile profile = new NicProfile(); + profile.setDeviceId(255); //dummyId + profile.setIp4Address(userIp.getAddress().toString()); + profile.setNetmask(publicIp.getNetmask()); + profile.setGateway(publicIp.getGateway()); + profile.setMacAddress(publicIp.getMacAddress()); + profile.setBroadcastType(network.getBroadcastDomainType()); + profile.setTrafficType(network.getTrafficType()); + profile.setBroadcastUri(broadcastUri); + profile.setIsolationUri(Networks.IsolationType.Vlan.toUri(publicIp.getVlanTag())); + profile.setSecurityGroupEnabled(_networkModel.isSecurityGroupSupportedInNetwork(network)); + profile.setName(_networkModel.getNetworkTag(vm.getHypervisorType(), network)); + profile.setNetworId(network.getId()); + + guru.updateNicProfile(profile, network); + vm.addNic(profile); + addedURIs.add(broadcastUri.toString()); + } + } + } + } + private NicProfile findNicProfileById(VirtualMachineProfile vm, long id) { for (NicProfile nic : vm.getNics()) { if (nic.getId() == id) { diff --git a/engine/schema/src/com/cloud/network/dao/IPAddressDao.java b/engine/schema/src/com/cloud/network/dao/IPAddressDao.java index 8d60b570414..cb076568b13 100755 --- a/engine/schema/src/com/cloud/network/dao/IPAddressDao.java +++ b/engine/schema/src/com/cloud/network/dao/IPAddressDao.java @@ -84,4 +84,5 @@ public interface IPAddressDao extends GenericDao { void lockRange(long vlandbId); + List listByAssociatedVmId(long vmId); } diff --git a/engine/schema/src/com/cloud/network/dao/IPAddressDaoImpl.java b/engine/schema/src/com/cloud/network/dao/IPAddressDaoImpl.java index 6f6bfd27d24..5122876adf8 100755 --- a/engine/schema/src/com/cloud/network/dao/IPAddressDaoImpl.java +++ b/engine/schema/src/com/cloud/network/dao/IPAddressDaoImpl.java @@ -430,6 +430,13 @@ public class IPAddressDaoImpl extends GenericDaoBase implemen return findOneBy(sc); } + @Override + public List listByAssociatedVmId(long vmId) { + SearchCriteria sc = AllFieldsSearch.create(); + sc.setParameters("associatedWithVmId", vmId); + return listBy(sc); + } + @Override public void lockRange(long vlandbId) { SearchCriteria sc = AllFieldsSearch.create(); diff --git a/server/src/com/cloud/hypervisor/HypervisorGuruBase.java b/server/src/com/cloud/hypervisor/HypervisorGuruBase.java index 308c327177f..0cb9af5bf8c 100644 --- a/server/src/com/cloud/hypervisor/HypervisorGuruBase.java +++ b/server/src/com/cloud/hypervisor/HypervisorGuruBase.java @@ -18,6 +18,7 @@ package com.cloud.hypervisor; import java.util.List; import java.util.Map; +import java.util.UUID; import javax.inject.Inject; @@ -112,6 +113,9 @@ public abstract class HypervisorGuruBase extends AdapterBase implements Hypervis to.setNicSecIps(secIps); } else { s_logger.warn("Unabled to load NicVO for NicProfile " + profile.getId()); + //Workaround for dynamically created nics + //FixMe: uuid and secondary IPs can be made part of nic profile + to.setUuid(UUID.randomUUID().toString()); } //check whether the this nic has secondary ip addresses set diff --git a/server/test/com/cloud/vpc/MockNetworkManagerImpl.java b/server/test/com/cloud/vpc/MockNetworkManagerImpl.java index 097019d76e8..e4c97a0558e 100644 --- a/server/test/com/cloud/vpc/MockNetworkManagerImpl.java +++ b/server/test/com/cloud/vpc/MockNetworkManagerImpl.java @@ -838,6 +838,11 @@ public class MockNetworkManagerImpl extends ManagerBase implements NetworkOrches return false; //To change body of implemented methods use File | Settings | File Templates. } + @Override + public void prepareAllNicsForMigration(VirtualMachineProfile vm, DeployDestination dest) { + return; + } + @Override public void prepareNicForMigration(VirtualMachineProfile vm, DeployDestination dest) { // TODO Auto-generated method stub From fd0eb6c29e09164da08039e686601b9d5a773287 Mon Sep 17 00:00:00 2001 From: Likitha Shetty Date: Mon, 5 Jan 2015 11:35:43 +0530 Subject: [PATCH 063/391] CLOUDSTACK-3439. [VMWARE] While preparing for migration, include dynamically created nics in case of VMware too. (cherry picked from commit 4b67a48144402a3a43d51c869456d9268873b1b1) Signed-off-by: Rohit Yadav --- .../cloudstack/engine/orchestration/NetworkOrchestrator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java b/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java index c84fb8d12cd..244ce472da0 100755 --- a/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java +++ b/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java @@ -1377,7 +1377,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra @Override public void prepareNicForMigration(VirtualMachineProfile vm, DeployDestination dest) { - if(vm.getType().equals(VirtualMachine.Type.DomainRouter) && vm.getHypervisorType().equals(HypervisorType.KVM)){ + if(vm.getType().equals(VirtualMachine.Type.DomainRouter) && (vm.getHypervisorType().equals(HypervisorType.KVM) || vm.getHypervisorType().equals(HypervisorType.VMware))) { //Include nics hot plugged and not stored in DB prepareAllNicsForMigration(vm, dest); return; From a1a601be5a08a42739c3ef999f551db1b7bd7749 Mon Sep 17 00:00:00 2001 From: Abhinandan Prateek Date: Mon, 19 Jan 2015 11:29:22 +0530 Subject: [PATCH 064/391] CLOUDSTACK-8164: Look for all host to avoid NPE Signed-off-by: Rohit Yadav --- .../apache/cloudstack/storage/helper/VMSnapshotHelperImpl.java | 2 +- server/src/com/cloud/vm/snapshot/VMSnapshotManagerImpl.java | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/engine/storage/src/org/apache/cloudstack/storage/helper/VMSnapshotHelperImpl.java b/engine/storage/src/org/apache/cloudstack/storage/helper/VMSnapshotHelperImpl.java index a1bcd45d3a2..cadbad3341a 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/helper/VMSnapshotHelperImpl.java +++ b/engine/storage/src/org/apache/cloudstack/storage/helper/VMSnapshotHelperImpl.java @@ -82,7 +82,7 @@ public class VMSnapshotHelperImpl implements VMSnapshotHelper { // check if lastHostId is available if (vm.getLastHostId() != null) { - HostVO lastHost = hostDao.findById(vm.getLastHostId()); + HostVO lastHost = hostDao.findByIdIncludingRemoved(vm.getLastHostId()); if (lastHost.getStatus() == com.cloud.host.Status.Up && !lastHost.isInMaintenanceStates()) return lastHost.getId(); } diff --git a/server/src/com/cloud/vm/snapshot/VMSnapshotManagerImpl.java b/server/src/com/cloud/vm/snapshot/VMSnapshotManagerImpl.java index 6590a748a31..e7df0886ccb 100644 --- a/server/src/com/cloud/vm/snapshot/VMSnapshotManagerImpl.java +++ b/server/src/com/cloud/vm/snapshot/VMSnapshotManagerImpl.java @@ -748,6 +748,8 @@ public class VMSnapshotManagerImpl extends ManagerBase implements VMSnapshotMana if (jobResult != null) { if (jobResult instanceof ConcurrentOperationException) throw (ConcurrentOperationException)jobResult; + else if (jobResult instanceof InvalidParameterValueException) + throw (InvalidParameterValueException)jobResult; else if (jobResult instanceof Throwable) throw new RuntimeException("Unexpected exception", (Throwable)jobResult); } From 960d91e0d54af13dccbc3ad0864b8630ac6d7082 Mon Sep 17 00:00:00 2001 From: abhi Date: Mon, 19 Jan 2015 15:44:14 +0530 Subject: [PATCH 065/391] CLOUDSTACK-8165: updating guest os id for xen template to corresponding 64 bit as default template is 64 bit --- setup/db/templates.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/db/templates.sql b/setup/db/templates.sql index 6a74633efc9..c42de21f0fc 100755 --- a/setup/db/templates.sql +++ b/setup/db/templates.sql @@ -16,7 +16,7 @@ -- under the License. INSERT INTO `cloud`.`vm_template` (id, uuid, unique_name, name, public, created, type, hvm, bits, account_id, url, checksum, enable_password, display_text, format, guest_os_id, featured, cross_zones, hypervisor_type) - VALUES (1, UUID(), 'routing-1', 'SystemVM Template (XenServer)', 0, now(), 'SYSTEM', 0, 64, 1, 'http://download.cloud.com/templates/4.3/systemvm64template-2014-01-14-master-xen.vhd.bz2', '74b92f031cc5c2089ee89efb81344dcf', 0, 'SystemVM Template (XenServer)', 'VHD', 183, 0, 1, 'XenServer' ); + VALUES (1, UUID(), 'routing-1', 'SystemVM Template (XenServer)', 0, now(), 'SYSTEM', 0, 64, 1, 'http://download.cloud.com/templates/4.3/systemvm64template-2014-01-14-master-xen.vhd.bz2', '74b92f031cc5c2089ee89efb81344dcf', 0, 'SystemVM Template (XenServer)', 'VHD', 184, 0, 1, 'XenServer' ); INSERT INTO `cloud`.`vm_template` (id, uuid, unique_name, name, public, created, removed, type, hvm, bits, account_id, url, checksum, enable_password, display_text, format, guest_os_id, featured, cross_zones, hypervisor_type, extractable) VALUES (2, UUID(), 'centos53-x86_64', 'CentOS 5.3(64-bit) no GUI (XenServer)', 1, now(), now(), 'BUILTIN', 0, 64, 1, 'http://download.cloud.com/templates/builtin/f59f18fb-ae94-4f97-afd2-f84755767aca.vhd.bz2', 'b63d854a9560c013142567bbae8d98cf', 0, 'CentOS 5.3(64-bit) no GUI (XenServer)', 'VHD', 12, 1, 1, 'XenServer', 1); From 0763b5576dde92b1c4ecfbce3b3b593ed0ca90bc Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Tue, 20 Jan 2015 07:47:34 +0530 Subject: [PATCH 066/391] CID 1264632: Rely on a known and widely available encoding UTF8 Signed-off-by: Rohit Yadav --- utils/src/com/cloud/utils/ConstantTimeComparator.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/utils/src/com/cloud/utils/ConstantTimeComparator.java b/utils/src/com/cloud/utils/ConstantTimeComparator.java index 4d4a595309a..baf2bc2738f 100644 --- a/utils/src/com/cloud/utils/ConstantTimeComparator.java +++ b/utils/src/com/cloud/utils/ConstantTimeComparator.java @@ -19,6 +19,8 @@ package com.cloud.utils; +import java.nio.charset.Charset; + public class ConstantTimeComparator { public static boolean compareBytes(byte[] b1, byte[] b2) { @@ -34,6 +36,7 @@ public class ConstantTimeComparator { } public static boolean compareStrings(String s1, String s2) { - return compareBytes(s1.getBytes(), s2.getBytes()); + final Charset encoding = Charset.forName("UTF-8"); + return compareBytes(s1.getBytes(encoding), s2.getBytes(encoding)); } } From f702097b3ebf72d5b2b0725e07330f75a3657f81 Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Tue, 20 Jan 2015 07:48:26 +0530 Subject: [PATCH 067/391] packaging: fix rpm package warning, fix day/date Signed-off-by: Rohit Yadav --- packaging/centos63/cloud.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/centos63/cloud.spec b/packaging/centos63/cloud.spec index 072a0b94884..66246479a7a 100644 --- a/packaging/centos63/cloud.spec +++ b/packaging/centos63/cloud.spec @@ -712,5 +712,5 @@ fi * Fri Jul 04 2014 Hugo Trippaers 4.5.0 - Add a package for the mysql ha module -* Fri Oct 03 2012 Hugo Trippaers 4.1.0 +* Wed Oct 03 2012 Hugo Trippaers 4.1.0 - new style spec file From fc502a97c2a80c57bcee9084a4247d20b439f295 Mon Sep 17 00:00:00 2001 From: Gaurav Aradhye Date: Tue, 13 Jan 2015 13:56:41 +0530 Subject: [PATCH 068/391] CLOUDSTACK-8152: Adding delay before creating snapshot/template from root disk after writing data to disk - test_vm_passwordenabled.py Signed-off-by: SrikanteswaraRao Talluri (cherry picked from commit d79837b67a481e618c8291152a9ecec299ff02dd) Signed-off-by: Rohit Yadav Conflicts: test/integration/component/test_vm_passwdenabled.py --- .../component/test_vm_passwdenabled.py | 98 ++++++++++++------- 1 file changed, 60 insertions(+), 38 deletions(-) diff --git a/test/integration/component/test_vm_passwdenabled.py b/test/integration/component/test_vm_passwdenabled.py index 1b556da298d..64e549b7746 100644 --- a/test/integration/component/test_vm_passwdenabled.py +++ b/test/integration/component/test_vm_passwdenabled.py @@ -14,24 +14,33 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -import marvin -from marvin.cloudstackTestCase import * -from marvin.cloudstackAPI import * -from marvin.sshClient import SshClient -from marvin.lib.utils import * -from marvin.lib.base import * -from marvin.lib.common import * +from marvin.cloudstackTestCase import cloudstackTestCase +from marvin.lib.utils import cleanup_resources +from marvin.lib.base import (Account, + ServiceOffering, + VirtualMachine, + EgressFireWallRule, + Template) +from marvin.lib.common import (get_domain, + get_zone, + get_template, + list_virtual_machines, + list_volumes) from nose.plugins.attrib import attr +import time _multiprocess_shared_ = True + + class Services: + """Test VM Life Cycle Services """ def __init__(self): self.services = { - "disk_offering":{ + "disk_offering": { "displaytext": "Small", "name": "Small", "disksize": 1 @@ -44,26 +53,26 @@ class Services: # Random characters are appended in create account to # ensure unique username generated each time "password": "password", - }, + }, "small": # Create a small virtual machine instance with disk offering { "displayname": "testserver", - "username": "root", # VM creds for SSH + "username": "root", # VM creds for SSH "password": "password", "ssh_port": 22, "hypervisor": 'XenServer', "privateport": 22, "publicport": 22, "protocol": 'TCP', - }, + }, "egress": { "name": 'web', "protocol": 'TCP', "startport": 80, "endport": 80, "cidrlist": '0.0.0.0/0', - }, + }, "service_offerings": { "small": @@ -75,19 +84,20 @@ class Services: "cpunumber": 1, "cpuspeed": 100, "memory": 256, - }, - }, + }, + }, "template": { "displaytext": "Cent OS Template", "name": "Cent OS Template", "passwordenabled": True, - }, + }, "sleep": 60, "timeout": 10, "ostype": 'CentOS 5.3 (64-bit)', # CentOS 5.3 (64-bit) } + class TestVMPasswordEnabled(cloudstackTestCase): @classmethod @@ -132,33 +142,39 @@ class TestVMPasswordEnabled(cloudstackTestCase): networkid = cls.virtual_machine.nic[0].networkid - # create egress rule to allow wget of my cloud-set-guest-password script + # create egress rule to allow wget of my cloud-set-guest-password + # script if zone.networktype.lower() == 'advanced': - EgressFireWallRule.create(cls.api_client, - networkid=networkid, - protocol=cls.services["egress"]["protocol"], - startport=cls.services["egress"]["startport"], - endport=cls.services["egress"]["endport"], - cidrlist=cls.services["egress"]["cidrlist"]) + EgressFireWallRule.create( + cls.api_client, + networkid=networkid, + protocol=cls.services["egress"]["protocol"], + startport=cls.services["egress"]["startport"], + endport=cls.services["egress"]["endport"], + cidrlist=cls.services["egress"]["cidrlist"]) cls.virtual_machine.password = cls.services["small"]["password"] ssh = cls.virtual_machine.get_ssh_client() - #below steps are required to get the new password from VR(reset password) - #http://cloudstack.org/dl/cloud-set-guest-password - #Copy this file to /etc/init.d - #chmod +x /etc/init.d/cloud-set-guest-password - #chkconfig --add cloud-set-guest-password + # below steps are required to get the new password from VR + # (reset password) + # http://cloudstack.org/dl/cloud-set-guest-password + # Copy this file to /etc/init.d + # chmod +x /etc/init.d/cloud-set-guest-password + # chkconfig --add cloud-set-guest-password cmds = [ "cd /etc/init.d;wget http://people.apache.org/~tsp/cloud-set-guest-password", "chmod +x /etc/init.d/cloud-set-guest-password", "chkconfig --add cloud-set-guest-password", - ] + ] for c in cmds: - result = ssh.execute(c) + ssh.execute(c) - #Stop virtual machine + # Adding delay of 120 sec to avoid data loss due to timing issue + time.sleep(120) + + # Stop virtual machine cls.virtual_machine.stop(cls.api_client) # Poll listVM to ensure VM is stopped properly @@ -199,7 +215,8 @@ class TestVMPasswordEnabled(cloudstackTestCase): cls.virtual_machine.id) cls.services["template"]["ostype"] = cls.services["ostype"] - #Create templates for Edit, Delete & update permissions testcases + cls.services["template"]["ispublic"] = True + # Create templates for Edit, Delete & update permissions testcases cls.pw_enabled_template = Template.create( cls.api_client, cls.services["template"], @@ -237,11 +254,18 @@ class TestVMPasswordEnabled(cloudstackTestCase): self.cleanup = [] def tearDown(self): - #Clean up, terminate the created instances + # Clean up, terminate the created instances cleanup_resources(self.apiclient, self.cleanup) return - @attr(tags = ["advanced", "advancedns", "smoke", "basic", "sg"], required_hardware="true") + @attr( + tags=[ + "advanced", + "advancedns", + "smoke", + "basic", + "sg"], + required_hardware="true") def test_11_get_vm_password(self): """Test get VM password for password enabled template""" @@ -255,9 +279,6 @@ class TestVMPasswordEnabled(cloudstackTestCase): self.debug("Stopping VM: %s" % self.vm.name) self.vm.stop(self.apiclient) - # Sleep to ensure VM is stopped properly - time.sleep(self.services["sleep"]) - self.debug("Resetting VM password for VM: %s" % self.vm.name) password = self.vm.resetPassword(self.apiclient) self.debug("Password reset to: %s" % password) @@ -282,7 +303,8 @@ class TestVMPasswordEnabled(cloudstackTestCase): try: self.debug("SSHing into VM: %s" % self.vm.ssh_ip) self.vm.password = password - ssh = self.vm.get_ssh_client() + self.vm.get_ssh_client() except Exception as e: - self.fail("SSH into VM: %s failed" % self.vm.ssh_ip) + self.fail("SSH into VM: %s failed: %s" % + (self.vm.ssh_ip, e)) return From 9a75a0cd221efc85a9d2b27fafdbf60995aa57eb Mon Sep 17 00:00:00 2001 From: Ashutosh K Date: Mon, 29 Dec 2014 03:34:37 -0800 Subject: [PATCH 069/391] CLOUDSTACK-8132: Fixed issue related to secondary storage count of template Signed-off-by: SrikanteswaraRao Talluri (cherry picked from commit f938a5e1c352971aa61e75a60935c735c3d3920c) Signed-off-by: Rohit Yadav --- test/integration/component/test_ss_limits.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/test/integration/component/test_ss_limits.py b/test/integration/component/test_ss_limits.py index c498302c003..d5ee0633080 100644 --- a/test/integration/component/test_ss_limits.py +++ b/test/integration/component/test_ss_limits.py @@ -126,7 +126,7 @@ class TestSecondaryStorageLimits(cloudstackTestCase): except Exception as e: return [FAIL, e] return [PASS, None] - + @data(ROOT_DOMAIN_ADMIN, CHILD_DOMAIN_ADMIN) @attr(tags = ["advanced"], required_hardware="true") def test_01_register_template(self, value): @@ -143,24 +143,28 @@ class TestSecondaryStorageLimits(cloudstackTestCase): response = self.setupAccount(value) self.assertEqual(response[0], PASS, response[1]) + apiclient = self.testClient.getUserApiClient( + UserName=self.account.name, + DomainName=self.account.domain) + builtin_info = get_builtin_template_info(self.apiclient, self.zone.id) self.services["template_2"]["url"] = builtin_info[0] self.services["template_2"]["hypervisor"] = builtin_info[1] self.services["template_2"]["format"] = builtin_info[2] try: - template = Template.register(self.apiclient, + template = Template.register(apiclient, self.services["template_2"], zoneid=self.zone.id, account=self.account.name, domainid=self.account.domainid, hypervisor=self.hypervisor) - template.download(self.apiclient) + template.download(apiclient) except Exception as e: self.fail("Failed to register template: %s" % e) - templates = Template.list(self.apiclient, + templates = Template.list(apiclient, templatefilter=\ self.services["template_2"]["templatefilter"], id=template.id) @@ -170,19 +174,19 @@ class TestSecondaryStorageLimits(cloudstackTestCase): templateSize = (templates[0].size / (1024**3)) expectedCount = templateSize response = matchResourceCount( - self.apiclient, expectedCount, + apiclient, expectedCount, RESOURCE_SECONDARY_STORAGE, accountid=self.account.id) self.assertEqual(response[0], PASS, response[1]) try: - template.delete(self.apiclient) + template.delete(apiclient) except Exception as e: self.fail("Failed to delete template: %s" % e) expectedCount = 0 response = matchResourceCount( - self.apiclient, expectedCount, + apiclient, expectedCount, RESOURCE_SECONDARY_STORAGE, accountid=self.account.id) self.assertEqual(response[0], PASS, response[1]) From 345fd54fc141e6dab48ba2d3c99ea38417d9087f Mon Sep 17 00:00:00 2001 From: Ashutosh K Date: Fri, 26 Dec 2014 12:24:29 +0530 Subject: [PATCH 070/391] CLOUDSTACK-8130: Fixed test_escalations_templates.py - Removed test case dependency on each other Signed-off-by: SrikanteswaraRao Talluri (cherry picked from commit 17da2e9ce9894b67c32306fb36d02786d34ad0d0) Signed-off-by: Rohit Yadav --- .../component/test_escalations_templates.py | 1001 +++++++++-------- 1 file changed, 509 insertions(+), 492 deletions(-) diff --git a/test/integration/component/test_escalations_templates.py b/test/integration/component/test_escalations_templates.py index 3dc24c1e7ba..a7787c7d03d 100644 --- a/test/integration/component/test_escalations_templates.py +++ b/test/integration/component/test_escalations_templates.py @@ -16,15 +16,23 @@ # under the License. # Import Local Modules -from marvin.cloudstackTestCase import * -from marvin.cloudstackAPI import * -from marvin.lib.utils import * -from marvin.lib.base import * -from marvin.lib.common import * +from marvin.cloudstackTestCase import cloudstackTestCase, unittest +from marvin.lib.utils import (cleanup_resources, + validateList) +from marvin.lib.base import (Account, + Zone, + Template, + Hypervisor) +from marvin.lib.common import (get_domain, + get_zone, + get_template, + list_os_types, + get_builtin_template_info) from marvin.codes import PASS from nose.plugins.attrib import attr import time + class TestTemplates(cloudstackTestCase): @classmethod @@ -36,23 +44,21 @@ class TestTemplates(cloudstackTestCase): cls.services = cls.testClient.getParsedTestDataConfig() # Get Domain, Zone, Template cls.domain = get_domain(cls.api_client) - cls.zone = get_zone(cls.api_client, cls.testClient.getZoneForTests()) + cls.zone = get_zone( + cls.api_client, + cls.testClient.getZoneForTests()) cls.template = get_template( - cls.api_client, - cls.zone.id, - cls.services["ostype"] - ) + cls.api_client, + cls.zone.id, + cls.services["ostype"] + ) cls.hypervisor = cls.testClient.getHypervisorInfo() cls.services['mode'] = cls.zone.networktype - cls.account = Account.create( - cls.api_client, - cls.services["account"], - domainid=cls.domain.id - ) - # Getting authentication for user in newly created Account - cls.user = cls.account.user[0] - cls.userapiclient = cls.testClient.getUserApiClient(cls.user.username, cls.domain.name) - cls._cleanup.append(cls.account) + + builtin_info = get_builtin_template_info(cls.api_client, cls.zone.id) + cls.services["privatetemplate"]["url"] = builtin_info[0] + cls.services["privatetemplate"]["hypervisor"] = builtin_info[1] + cls.services["privatetemplate"]["format"] = builtin_info[2] except Exception as e: cls.tearDownClass() raise Exception("Warning: Exception in setup : %s" % e) @@ -62,6 +68,17 @@ class TestTemplates(cloudstackTestCase): self.apiClient = self.testClient.getApiClient() self.cleanup = [] + self.account = Account.create( + self.apiClient, + self.services["account"], + domainid=self.domain.id + ) + # Getting authentication for user in newly created Account + self.user = self.account.user[0] + self.userapiclient = self.testClient.getUserApiClient( + self.user.username, + self.domain.name) + self.cleanup.append(self.account) def tearDown(self): # Clean up, terminate the created resources @@ -102,10 +119,9 @@ class TestTemplates(cloudstackTestCase): return_flag = return_flag and True else: return_flag = return_flag and False - self.debug("expected Value: %s, is not matching with actual value: %s" % ( - exp_val, - act_val - )) + self.debug( + "expected Value: %s, is not matching with actual value: %s" % + (exp_val, act_val)) return return_flag @attr(tags=["advanced", "basic"], required_hardware="true") @@ -132,105 +148,104 @@ class TestTemplates(cloudstackTestCase): """ # Listing all the Templates for a User list_templates_before = Template.list( - self.userapiclient, - listall=self.services["listall"], - templatefilter=self.services["templatefilter"] - ) + self.userapiclient, + listall=self.services["listall"], + templatefilter=self.services["templatefilter"] + ) # Verifying that no Templates are listed self.assertIsNone( - list_templates_before, - "Templates listed for newly created User" - ) - self.services["templateregister"]["ostype"] = self.services["ostype"] + list_templates_before, + "Templates listed for newly created User" + ) + self.services["privatetemplate"]["ostype"] = self.services["ostype"] # Creating pagesize + 1 number of Templates for i in range(0, (self.services["pagesize"] + 1)): template_created = Template.register( - self.userapiclient, - self.services["templateregister"], - self.zone.id, - hypervisor=self.hypervisor - ) + self.userapiclient, + self.services["privatetemplate"], + self.zone.id, + hypervisor=self.hypervisor + ) self.assertIsNotNone( - template_created, - "Template creation failed" - ) - if(i < self.services["pagesize"]): - self.cleanup.append(template_created) + template_created, + "Template creation failed" + ) # Listing all the Templates for a User list_templates_after = Template.list( - self.userapiclient, - listall=self.services["listall"], - templatefilter=self.services["templatefilter"] - ) + self.userapiclient, + listall=self.services["listall"], + templatefilter=self.services["templatefilter"] + ) status = validateList(list_templates_after) self.assertEquals( - PASS, - status[0], - "Templates creation failed" - ) + PASS, + status[0], + "Templates creation failed" + ) # Verifying that list size is pagesize + 1 self.assertEquals( - self.services["pagesize"] + 1, - len(list_templates_after), - "Failed to create pagesize + 1 number of Templates" - ) + self.services["pagesize"] + 1, + len(list_templates_after), + "Failed to create pagesize + 1 number of Templates" + ) # Listing all the Templates in page 1 list_templates_page1 = Template.list( - self.userapiclient, - listall=self.services["listall"], - templatefilter=self.services["templatefilter"], - page=1, - pagesize=self.services["pagesize"] - ) + self.userapiclient, + listall=self.services["listall"], + templatefilter=self.services["templatefilter"], + page=1, + pagesize=self.services["pagesize"] + ) status = validateList(list_templates_page1) self.assertEquals( - PASS, - status[0], - "Failed to list Templates in page 1" - ) + PASS, + status[0], + "Failed to list Templates in page 1" + ) # Verifying the list size to be equal to pagesize self.assertEquals( - self.services["pagesize"], - len(list_templates_page1), - "Size of Templates in page 1 is not matching" - ) + self.services["pagesize"], + len(list_templates_page1), + "Size of Templates in page 1 is not matching" + ) # Listing all the Templates in page 2 list_templates_page2 = Template.list( - self.userapiclient, - listall=self.services["listall"], - templatefilter=self.services["templatefilter"], - page=2, - pagesize=self.services["pagesize"] - ) + self.userapiclient, + listall=self.services["listall"], + templatefilter=self.services["templatefilter"], + page=2, + pagesize=self.services["pagesize"] + ) status = validateList(list_templates_page2) self.assertEquals( - PASS, - status[0], - "Failed to list Templates in page 2" - ) + PASS, + status[0], + "Failed to list Templates in page 2" + ) # Verifying the list size to be equal to 1 self.assertEquals( - 1, - len(list_templates_page2), - "Size of Templates in page 2 is not matching" - ) - # Verifying the state of the template to be ready. If not waiting for state to become ready + 1, + len(list_templates_page2), + "Size of Templates in page 2 is not matching" + ) + # Verifying the state of the template to be ready. If not waiting for + # state to become ready template_ready = False count = 0 while template_ready is False: list_template = Template.list( - self.userapiclient, - id=template_created.id, - listall=self.services["listall"], - templatefilter=self.services["templatefilter"], - ) + self.userapiclient, + id=template_created.id, + listall=self.services["listall"], + templatefilter=self.services["templatefilter"], + ) status = validateList(list_template) self.assertEquals( - PASS, - status[0], - "Failed to list Templates by Id" - ) + PASS, + status[0], + "Failed to list Templates by Id" + ) if list_template[0].isready is True: template_ready = True elif (str(list_template[0].status) == "Error"): @@ -245,23 +260,23 @@ class TestTemplates(cloudstackTestCase): # Deleting the Template present in page 2 Template.delete( - template_created, - self.userapiclient - ) + template_created, + self.userapiclient + ) # Listing all the Templates in page 2 again list_templates_page2 = Template.list( - self.userapiclient, - listall=self.services["listall"], - templatefilter=self.services["templatefilter"], - page=2, - pagesize=self.services["pagesize"] - ) + self.userapiclient, + listall=self.services["listall"], + templatefilter=self.services["templatefilter"], + page=2, + pagesize=self.services["pagesize"] + ) # Verifying that there are no Templates listed self.assertIsNone( - list_templates_page2, - "Templates not deleted from page 2" - ) - del self.services["templateregister"]["ostype"] + list_templates_page2, + "Templates not deleted from page 2" + ) + del self.services["privatetemplate"]["ostype"] return @attr(tags=["advanced", "basic"], required_hardware="true") @@ -282,63 +297,63 @@ class TestTemplates(cloudstackTestCase): """ # Listing all the Templates for a User list_templates_before = Template.list( - self.userapiclient, - listall=self.services["listall"], - templatefilter=self.services["templatefilter"] - ) + self.userapiclient, + listall=self.services["listall"], + templatefilter=self.services["templatefilter"] + ) # Verifying that no Templates are listed self.assertIsNone( - list_templates_before, - "Templates listed for newly created User" - ) - self.services["templateregister"]["ostype"] = self.services["ostype"] - self.services["templateregister"]["isextractable"] = True + list_templates_before, + "Templates listed for newly created User" + ) + self.services["privatetemplate"]["ostype"] = self.services["ostype"] + self.services["privatetemplate"]["isextractable"] = True # Creating aTemplate template_created = Template.register( - self.userapiclient, - self.services["templateregister"], - self.zone.id, - hypervisor=self.hypervisor - ) + self.userapiclient, + self.services["privatetemplate"], + self.zone.id, + hypervisor=self.hypervisor + ) self.assertIsNotNone( - template_created, - "Template creation failed" - ) - self.cleanup.append(template_created) + template_created, + "Template creation failed" + ) # Listing all the Templates for a User list_templates_after = Template.list( - self.userapiclient, - listall=self.services["listall"], - templatefilter=self.services["templatefilter"] - ) + self.userapiclient, + listall=self.services["listall"], + templatefilter=self.services["templatefilter"] + ) status = validateList(list_templates_after) self.assertEquals( - PASS, - status[0], - "Templates creation failed" - ) + PASS, + status[0], + "Templates creation failed" + ) # Verifying that list size is 1 self.assertEquals( - 1, - len(list_templates_after), - "Failed to create a Template" - ) - # Verifying the state of the template to be ready. If not waiting for state to become ready till time out + 1, + len(list_templates_after), + "Failed to create a Template" + ) + # Verifying the state of the template to be ready. If not waiting for + # state to become ready till time out template_ready = False count = 0 while template_ready is False: list_template = Template.list( - self.userapiclient, - id=template_created.id, - listall=self.services["listall"], - templatefilter=self.services["templatefilter"], - ) + self.userapiclient, + id=template_created.id, + listall=self.services["listall"], + templatefilter=self.services["templatefilter"], + ) status = validateList(list_template) self.assertEquals( - PASS, - status[0], - "Failed to list Templates by Id" - ) + PASS, + status[0], + "Failed to list Templates by Id" + ) if list_template[0].isready is True: template_ready = True elif (str(list_template[0].status) == "Error"): @@ -353,32 +368,32 @@ class TestTemplates(cloudstackTestCase): # Downloading the Template name download_template = Template.extract( - self.userapiclient, - template_created.id, - mode="HTTP_DOWNLOAD", - zoneid=self.zone.id - ) + self.userapiclient, + template_created.id, + mode="HTTP_DOWNLOAD", + zoneid=self.zone.id + ) self.assertIsNotNone( - download_template, - "Download Template failed" - ) - # Verifying the details of downloaded template + download_template, + "Download Template failed" + ) + # Verifying the details of downloaded template self.assertEquals( - "DOWNLOAD_URL_CREATED", - download_template.state, - "Download URL not created for Template" - ) + "DOWNLOAD_URL_CREATED", + download_template.state, + "Download URL not created for Template" + ) self.assertIsNotNone( - download_template.url, - "Download URL not created for Template" - ) + download_template.url, + "Download URL not created for Template" + ) self.assertEquals( - template_created.id, - download_template.id, - "Download Template details are not same as Template created" - ) - del self.services["templateregister"]["ostype"] - del self.services["templateregister"]["isextractable"] + template_created.id, + download_template.id, + "Download Template details are not same as Template created" + ) + del self.services["privatetemplate"]["ostype"] + del self.services["privatetemplate"]["isextractable"] return @attr(tags=["advanced", "basic"], required_hardware="true") @@ -407,62 +422,62 @@ class TestTemplates(cloudstackTestCase): """ # Listing all the Templates for a User list_templates_before = Template.list( - self.userapiclient, - listall=self.services["listall"], - templatefilter=self.services["templatefilter"] - ) + self.userapiclient, + listall=self.services["listall"], + templatefilter=self.services["templatefilter"] + ) # Verifying that no Templates are listed self.assertIsNone( - list_templates_before, - "Templates listed for newly created User" - ) - self.services["templateregister"]["ostype"] = self.services["ostype"] + list_templates_before, + "Templates listed for newly created User" + ) + self.services["privatetemplate"]["ostype"] = self.services["ostype"] # Creating aTemplate template_created = Template.register( - self.userapiclient, - self.services["templateregister"], - self.zone.id, - hypervisor=self.hypervisor - ) + self.userapiclient, + self.services["privatetemplate"], + self.zone.id, + hypervisor=self.hypervisor + ) self.assertIsNotNone( - template_created, - "Template creation failed" - ) - self.cleanup.append(template_created) + template_created, + "Template creation failed" + ) # Listing all the Templates for a User list_templates_after = Template.list( - self.userapiclient, - listall=self.services["listall"], - templatefilter=self.services["templatefilter"] - ) + self.userapiclient, + listall=self.services["listall"], + templatefilter=self.services["templatefilter"] + ) status = validateList(list_templates_after) self.assertEquals( - PASS, - status[0], - "Templates creation failed" - ) + PASS, + status[0], + "Templates creation failed" + ) # Verifying that list size is 1 self.assertEquals( - 1, - len(list_templates_after), - "Failed to create a Template" - ) - # Verifying the state of the template to be ready. If not waiting for state to become ready till time out + 1, + len(list_templates_after), + "Failed to create a Template" + ) + # Verifying the state of the template to be ready. If not waiting for + # state to become ready till time out template_ready = False count = 0 while template_ready is False: list_template = Template.list( - self.userapiclient, - id=template_created.id, - listall=self.services["listall"], - templatefilter=self.services["templatefilter"], - ) + self.userapiclient, + id=template_created.id, + listall=self.services["listall"], + templatefilter=self.services["templatefilter"], + ) status = validateList(list_template) self.assertEquals( - PASS, - status[0], - "Failed to list Templates by Id" - ) + PASS, + status[0], + "Failed to list Templates by Id" + ) if list_template[0].isready is True: template_ready = True elif (str(list_template[0].status) == "Error"): @@ -477,220 +492,220 @@ class TestTemplates(cloudstackTestCase): # Editing the Template name edited_template = Template.update( - template_created, - self.userapiclient, - name="NewTemplateName" - ) + template_created, + self.userapiclient, + name="NewTemplateName" + ) self.assertIsNotNone( - edited_template, - "Editing Template failed" - ) - # Verifying the details of edited template + edited_template, + "Editing Template failed" + ) + # Verifying the details of edited template expected_dict = { - "id":template_created.id, - "name":"NewTemplateName", - "displaytest":template_created.displaytext, - "account":template_created.account, - "domainid":template_created.domainid, - "format":template_created.format, - "ostypeid":template_created.ostypeid, - "templatetype":template_created.templatetype, - } + "id": template_created.id, + "name": "NewTemplateName", + "displaytest": template_created.displaytext, + "account": template_created.account, + "domainid": template_created.domainid, + "format": template_created.format, + "ostypeid": template_created.ostypeid, + "templatetype": template_created.templatetype, + } actual_dict = { - "id":edited_template.id, - "name":edited_template.name, - "displaytest":edited_template.displaytext, - "account":edited_template.account, - "domainid":edited_template.domainid, - "format":edited_template.format, - "ostypeid":edited_template.ostypeid, - "templatetype":edited_template.templatetype, - } + "id": edited_template.id, + "name": edited_template.name, + "displaytest": edited_template.displaytext, + "account": edited_template.account, + "domainid": edited_template.domainid, + "format": edited_template.format, + "ostypeid": edited_template.ostypeid, + "templatetype": edited_template.templatetype, + } edit_template_status = self.__verify_values( - expected_dict, - actual_dict - ) + expected_dict, + actual_dict + ) self.assertEqual( - True, - edit_template_status, - "Edited Template details are not as expected" - ) + True, + edit_template_status, + "Edited Template details are not as expected" + ) # Editing the Template displaytext edited_template = Template.update( - template_created, - self.userapiclient, - displaytext="TemplateDisplaytext" - ) + template_created, + self.userapiclient, + displaytext="TemplateDisplaytext" + ) self.assertIsNotNone( - edited_template, - "Editing Template failed" - ) - # Verifying the details of edited template + edited_template, + "Editing Template failed" + ) + # Verifying the details of edited template expected_dict = { - "id":template_created.id, - "name":"NewTemplateName", - "displaytest":"TemplateDisplaytext", - "account":template_created.account, - "domainid":template_created.domainid, - "format":template_created.format, - "ostypeid":template_created.ostypeid, - "templatetype":template_created.templatetype, - } + "id": template_created.id, + "name": "NewTemplateName", + "displaytest": "TemplateDisplaytext", + "account": template_created.account, + "domainid": template_created.domainid, + "format": template_created.format, + "ostypeid": template_created.ostypeid, + "templatetype": template_created.templatetype, + } actual_dict = { - "id":edited_template.id, - "name":edited_template.name, - "displaytest":edited_template.displaytext, - "account":edited_template.account, - "domainid":edited_template.domainid, - "format":edited_template.format, - "ostypeid":edited_template.ostypeid, - "templatetype":edited_template.templatetype, - } + "id": edited_template.id, + "name": edited_template.name, + "displaytest": edited_template.displaytext, + "account": edited_template.account, + "domainid": edited_template.domainid, + "format": edited_template.format, + "ostypeid": edited_template.ostypeid, + "templatetype": edited_template.templatetype, + } edit_template_status = self.__verify_values( - expected_dict, - actual_dict - ) + expected_dict, + actual_dict + ) self.assertEqual( - True, - edit_template_status, - "Edited Template details are not as expected" - ) + True, + edit_template_status, + "Edited Template details are not as expected" + ) # Editing the Template ostypeid ostype_list = list_os_types(self.userapiclient) status = validateList(ostype_list) self.assertEquals( - PASS, - status[0], - "Failed to list OS Types" - ) + PASS, + status[0], + "Failed to list OS Types" + ) for i in range(0, len(ostype_list)): if ostype_list[i].id != template_created.ostypeid: newostypeid = ostype_list[i].id break edited_template = Template.update( - template_created, - self.userapiclient, - ostypeid=newostypeid - ) + template_created, + self.userapiclient, + ostypeid=newostypeid + ) self.assertIsNotNone( - edited_template, - "Editing Template failed" - ) - # Verifying the details of edited template + edited_template, + "Editing Template failed" + ) + # Verifying the details of edited template expected_dict = { - "id":template_created.id, - "name":"NewTemplateName", - "displaytest":"TemplateDisplaytext", - "account":template_created.account, - "domainid":template_created.domainid, - "format":template_created.format, - "ostypeid":newostypeid, - "templatetype":template_created.templatetype, - } + "id": template_created.id, + "name": "NewTemplateName", + "displaytest": "TemplateDisplaytext", + "account": template_created.account, + "domainid": template_created.domainid, + "format": template_created.format, + "ostypeid": newostypeid, + "templatetype": template_created.templatetype, + } actual_dict = { - "id":edited_template.id, - "name":edited_template.name, - "displaytest":edited_template.displaytext, - "account":edited_template.account, - "domainid":edited_template.domainid, - "format":edited_template.format, - "ostypeid":edited_template.ostypeid, - "templatetype":edited_template.templatetype, - } + "id": edited_template.id, + "name": edited_template.name, + "displaytest": edited_template.displaytext, + "account": edited_template.account, + "domainid": edited_template.domainid, + "format": edited_template.format, + "ostypeid": edited_template.ostypeid, + "templatetype": edited_template.templatetype, + } edit_template_status = self.__verify_values( - expected_dict, - actual_dict - ) + expected_dict, + actual_dict + ) self.assertEqual( - True, - edit_template_status, - "Edited Template details are not as expected" - ) + True, + edit_template_status, + "Edited Template details are not as expected" + ) # Editing the Template name, displaytext edited_template = Template.update( - template_created, - self.userapiclient, - name=template_created.name, - displaytext=template_created.displaytext - ) + template_created, + self.userapiclient, + name=template_created.name, + displaytext=template_created.displaytext + ) self.assertIsNotNone( - edited_template, - "Editing Template failed" - ) - # Verifying the details of edited template + edited_template, + "Editing Template failed" + ) + # Verifying the details of edited template expected_dict = { - "id":template_created.id, - "name":template_created.name, - "displaytest":template_created.displaytext, - "account":template_created.account, - "domainid":template_created.domainid, - "format":template_created.format, - "ostypeid":newostypeid, - "templatetype":template_created.templatetype, - } + "id": template_created.id, + "name": template_created.name, + "displaytest": template_created.displaytext, + "account": template_created.account, + "domainid": template_created.domainid, + "format": template_created.format, + "ostypeid": newostypeid, + "templatetype": template_created.templatetype, + } actual_dict = { - "id":edited_template.id, - "name":edited_template.name, - "displaytest":edited_template.displaytext, - "account":edited_template.account, - "domainid":edited_template.domainid, - "format":edited_template.format, - "ostypeid":edited_template.ostypeid, - "templatetype":edited_template.templatetype, - } + "id": edited_template.id, + "name": edited_template.name, + "displaytest": edited_template.displaytext, + "account": edited_template.account, + "domainid": edited_template.domainid, + "format": edited_template.format, + "ostypeid": edited_template.ostypeid, + "templatetype": edited_template.templatetype, + } edit_template_status = self.__verify_values( - expected_dict, - actual_dict - ) + expected_dict, + actual_dict + ) self.assertEqual( - True, - edit_template_status, - "Edited Template details are not as expected" - ) + True, + edit_template_status, + "Edited Template details are not as expected" + ) # Editing the Template name, displaytext, ostypeid edited_template = Template.update( - template_created, - self.userapiclient, - name="NewTemplateName", - displaytext="TemplateDisplaytext", - ostypeid=template_created.ostypeid - ) + template_created, + self.userapiclient, + name="NewTemplateName", + displaytext="TemplateDisplaytext", + ostypeid=template_created.ostypeid + ) self.assertIsNotNone( - edited_template, - "Editing Template failed" - ) - # Verifying the details of edited template + edited_template, + "Editing Template failed" + ) + # Verifying the details of edited template expected_dict = { - "id":template_created.id, - "name":"NewTemplateName", - "displaytest":"TemplateDisplaytext", - "account":template_created.account, - "domainid":template_created.domainid, - "format":template_created.format, - "ostypeid":template_created.ostypeid, - "templatetype":template_created.templatetype, - } + "id": template_created.id, + "name": "NewTemplateName", + "displaytest": "TemplateDisplaytext", + "account": template_created.account, + "domainid": template_created.domainid, + "format": template_created.format, + "ostypeid": template_created.ostypeid, + "templatetype": template_created.templatetype, + } actual_dict = { - "id":edited_template.id, - "name":edited_template.name, - "displaytest":edited_template.displaytext, - "account":edited_template.account, - "domainid":edited_template.domainid, - "format":edited_template.format, - "ostypeid":edited_template.ostypeid, - "templatetype":edited_template.templatetype, - } + "id": edited_template.id, + "name": edited_template.name, + "displaytest": edited_template.displaytext, + "account": edited_template.account, + "domainid": edited_template.domainid, + "format": edited_template.format, + "ostypeid": edited_template.ostypeid, + "templatetype": edited_template.templatetype, + } edit_template_status = self.__verify_values( - expected_dict, - actual_dict - ) + expected_dict, + actual_dict + ) self.assertEqual( - True, - edit_template_status, - "Edited Template details are not as expected" - ) - del self.services["templateregister"]["ostype"] + True, + edit_template_status, + "Edited Template details are not as expected" + ) + del self.services["privatetemplate"]["ostype"] return @attr(tags=["advanced", "basic"], required_hardware="true") @@ -719,120 +734,122 @@ class TestTemplates(cloudstackTestCase): """ # Listing Zones available for a user zones_list = Zone.list( - self.userapiclient, - available=True - ) + self.userapiclient, + available=True + ) status = validateList(zones_list) self.assertEquals( - PASS, - status[0], - "Failed to list Zones" - ) + PASS, + status[0], + "Failed to list Zones" + ) if not len(zones_list) > 1: raise unittest.SkipTest("Not enough zones exist to copy template") else: # Listing all the Templates for a User in Zone 1 list_templates_zone1 = Template.list( - self.userapiclient, - listall=self.services["listall"], - templatefilter=self.services["templatefilter"], - zoneid=zones_list[0].id - ) + self.userapiclient, + listall=self.services["listall"], + templatefilter=self.services["templatefilter"], + zoneid=zones_list[0].id + ) # Verifying that no Templates are listed self.assertIsNone( - list_templates_zone1, - "Templates listed for newly created User in Zone1" - ) + list_templates_zone1, + "Templates listed for newly created User in Zone1" + ) # Listing all the Templates for a User in Zone 2 list_templates_zone2 = Template.list( - self.userapiclient, - listall=self.services["listall"], - templatefilter=self.services["templatefilter"], - zoneid=zones_list[1].id - ) + self.userapiclient, + listall=self.services["listall"], + templatefilter=self.services["templatefilter"], + zoneid=zones_list[1].id + ) # Verifying that no Templates are listed self.assertIsNone( - list_templates_zone2, - "Templates listed for newly created User in Zone2" - ) - self.services["templateregister"]["ostype"] = self.services["ostype"] + list_templates_zone2, + "Templates listed for newly created User in Zone2" + ) + self.services["privatetemplate"][ + "ostype"] = self.services["ostype"] # Listing Hypervisors in Zone 1 hypervisor_list = Hypervisor.list( - self.apiClient, - zoneid=zones_list[0].id - ) + self.apiClient, + zoneid=zones_list[0].id + ) status = validateList(zones_list) self.assertEquals( - PASS, - status[0], - "Failed to list Hypervisors in Zone 1" - ) + PASS, + status[0], + "Failed to list Hypervisors in Zone 1" + ) # Creating aTemplate in Zone 1 template_created = Template.register( - self.userapiclient, - self.services["templateregister"], - zones_list[0].id, - hypervisor=hypervisor_list[0].name - ) + self.userapiclient, + self.services["privatetemplate"], + zones_list[0].id, + hypervisor=hypervisor_list[0].name + ) self.assertIsNotNone( - template_created, - "Template creation failed" - ) - self.cleanup.append(template_created) + template_created, + "Template creation failed" + ) # Listing all the Templates for a User in Zone 1 list_templates_zone1 = Template.list( - self.userapiclient, - listall=self.services["listall"], - templatefilter=self.services["templatefilter"], - zoneid=zones_list[0].id - ) + self.userapiclient, + listall=self.services["listall"], + templatefilter=self.services["templatefilter"], + zoneid=zones_list[0].id + ) status = validateList(list_templates_zone1) self.assertEquals( - PASS, - status[0], - "Templates creation failed in Zone1" - ) + PASS, + status[0], + "Templates creation failed in Zone1" + ) # Verifying that list size is 1 self.assertEquals( - 1, - len(list_templates_zone1), - "Failed to create a Template" - ) + 1, + len(list_templates_zone1), + "Failed to create a Template" + ) # Listing all the Templates for a User in Zone 2 list_templates_zone2 = Template.list( - self.userapiclient, - listall=self.services["listall"], - templatefilter=self.services["templatefilter"], - zoneid=zones_list[1].id - ) + self.userapiclient, + listall=self.services["listall"], + templatefilter=self.services["templatefilter"], + zoneid=zones_list[1].id + ) # Verifying that no Templates are listed self.assertIsNone( - list_templates_zone2, - "Templates listed for newly created User in Zone2" - ) - # Verifying the state of the template to be ready. If not waiting for state to become ready till time out + list_templates_zone2, + "Templates listed for newly created User in Zone2" + ) + # Verifying the state of the template to be ready. If not waiting + # for state to become ready till time out template_ready = False count = 0 while template_ready is False: list_template = Template.list( - self.userapiclient, - id=template_created.id, - listall=self.services["listall"], - templatefilter=self.services["templatefilter"], - ) + self.userapiclient, + id=template_created.id, + listall=self.services["listall"], + templatefilter=self.services["templatefilter"], + ) status = validateList(list_template) self.assertEquals( - PASS, - status[0], - "Failed to list Templates by Id" - ) + PASS, + status[0], + "Failed to list Templates by Id" + ) if list_template[0].isready is True: template_ready = True elif (str(list_template[0].status) == "Error"): self.fail("Created Template is in Errored state") break elif count > 10: - self.fail("Timed out before Template came into ready state") + self.fail( + "Timed out before Template came into ready state") break else: time.sleep(self.services["sleep"]) @@ -840,61 +857,61 @@ class TestTemplates(cloudstackTestCase): # Copying the Template from Zone1 to Zone2 copied_template = template_created.copy( - self.userapiclient, - sourcezoneid=template_created.zoneid, - destzoneid=zones_list[1].id - ) + self.userapiclient, + sourcezoneid=template_created.zoneid, + destzoneid=zones_list[1].id + ) self.assertIsNotNone( - copied_template, - "Copying Template from Zone1 to Zone2 failed" - ) + copied_template, + "Copying Template from Zone1 to Zone2 failed" + ) # Listing all the Templates for a User in Zone 1 list_templates_zone1 = Template.list( - self.userapiclient, - listall=self.services["listall"], - templatefilter=self.services["templatefilter"], - zoneid=zones_list[0].id - ) + self.userapiclient, + listall=self.services["listall"], + templatefilter=self.services["templatefilter"], + zoneid=zones_list[0].id + ) status = validateList(list_templates_zone1) self.assertEquals( - PASS, - status[0], - "Templates creation failed in Zone1" - ) + PASS, + status[0], + "Templates creation failed in Zone1" + ) # Verifying that list size is 1 self.assertEquals( - 1, - len(list_templates_zone1), - "Failed to create a Template" - ) + 1, + len(list_templates_zone1), + "Failed to create a Template" + ) # Listing all the Templates for a User in Zone 2 list_templates_zone2 = Template.list( - self.userapiclient, - listall=self.services["listall"], - templatefilter=self.services["templatefilter"], - zoneid=zones_list[1].id - ) + self.userapiclient, + listall=self.services["listall"], + templatefilter=self.services["templatefilter"], + zoneid=zones_list[1].id + ) status = validateList(list_templates_zone2) self.assertEquals( - PASS, - status[0], - "Template failed to copy into Zone2" - ) + PASS, + status[0], + "Template failed to copy into Zone2" + ) # Verifying that list size is 1 self.assertEquals( - 1, - len(list_templates_zone2), - "Template failed to copy into Zone2" - ) + 1, + len(list_templates_zone2), + "Template failed to copy into Zone2" + ) self.assertNotEquals( - "Connection refused", - list_templates_zone2[0].status, - "Failed to copy Template" - ) + "Connection refused", + list_templates_zone2[0].status, + "Failed to copy Template" + ) self.assertEquals( - True, - list_templates_zone2[0].isready, - "Failed to copy Template" - ) - del self.services["templateregister"]["ostype"] + True, + list_templates_zone2[0].isready, + "Failed to copy Template" + ) + del self.services["privatetemplate"]["ostype"] return From 899bb8b2e92598228a1432760b02e6dfd5a4c8f0 Mon Sep 17 00:00:00 2001 From: Koushik Das Date: Tue, 23 Dec 2014 16:12:39 +0530 Subject: [PATCH 071/391] CLOUDSTACK-8115: Update default ordering of HA investigators Moved HV specific investigators before PingInvestigator in default ordering (cherry picked from commit ceae97868cb98b503324afb7743a25ce1f5c1516) Signed-off-by: Rohit Yadav --- .../cloudstack/core/spring-core-registry-core-context.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/resources/META-INF/cloudstack/core/spring-core-registry-core-context.xml b/core/resources/META-INF/cloudstack/core/spring-core-registry-core-context.xml index 3263e923229..939cffebce9 100644 --- a/core/resources/META-INF/cloudstack/core/spring-core-registry-core-context.xml +++ b/core/resources/META-INF/cloudstack/core/spring-core-registry-core-context.xml @@ -68,7 +68,7 @@ class="org.apache.cloudstack.spring.lifecycle.registry.ExtensionRegistry"> + value="SimpleInvestigator,XenServerInvestigator,KVMInvestigator,HypervInvestigator,VMwareInvestigator,PingInvestigator,ManagementIPSysVMInvestigator" /> From e6620b12e36a4dc95629435e77970d91727fd929 Mon Sep 17 00:00:00 2001 From: Sanjay Tripathi Date: Wed, 10 Dec 2014 14:16:48 +0530 Subject: [PATCH 072/391] CLOUDSTACK-8056: EN: Miss SC and UK keyboard option for VMware hypervisor when register a template. (cherry picked from commit 4d78703522d37980d7d3461b8bcaaa4dcdbaf4e0) Signed-off-by: Rohit Yadav --- ui/scripts/templates.js | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/ui/scripts/templates.js b/ui/scripts/templates.js index b95bb7dd759..1cb613b2672 100644 --- a/ui/scripts/templates.js +++ b/ui/scripts/templates.js @@ -325,11 +325,19 @@ }); items.push({ id: "us", - description: "US" + description: "US Keboard" + }); + items.push({ + id: "uk", + description: "UK Keyboard" }); items.push({ id: "jp", - description: "Japanese" + description: "Japanese Keyboard" + }); + items.push({ + id: "sc", + description: "Simplified Chinese" }); args.response.success({ data: items From cc3157130759909487aad17e6467b309cdebf132 Mon Sep 17 00:00:00 2001 From: SrikanteswaraRao Talluri Date: Wed, 17 Dec 2014 18:50:39 +0530 Subject: [PATCH 073/391] bug-id:CLOUDSTACK-8055cleaned up test tags, removed unecessary tags. reviewed-by: SrikanteswaraRao Talluri Signed-off-by: SrikanteswaraRao Talluri (cherry picked from commit b6bac7f67335454b6146ebf56b1c6a47842107c1) Signed-off-by: Rohit Yadav --- test/integration/component/test_portable_ip.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/component/test_portable_ip.py b/test/integration/component/test_portable_ip.py index 83ef6b51907..32e7b758221 100644 --- a/test/integration/component/test_portable_ip.py +++ b/test/integration/component/test_portable_ip.py @@ -407,7 +407,7 @@ class TestListPortablePublicIpRanges(cloudstackTestCase): "Listed netmask not matching with the netmask of created public ip range") return - @attr(tags=["advanced","swamy"], required_hardware="false") + @attr(tags=["advanced"], required_hardware="false") def test_list_portable_ip_range_non_root_admin(self): """Test list portable ip ranges with non admin root account """ @@ -1288,7 +1288,7 @@ class TestPortableIpTransferAcrossNetworks(cloudstackTestCase): raise Exception("Warning: Exception during cleanup : %s" % e) return - @attr(tags=["advanced","swamy"], required_hardware="true") + @attr(tags=["advanced"], required_hardware="true") def test_list_portable_ip_range_non_root_admin(self): """Test list portable ip ranges with non admin root account """ From 37cdb4eccacca005bc8a301921da96bec2f0838d Mon Sep 17 00:00:00 2001 From: Koushik Das Date: Wed, 10 Dec 2014 10:18:34 +0530 Subject: [PATCH 074/391] CLOUDSTACK-8054: No event generated on host maintenance mode is completed Added events for host maintenance start and completion (cherry picked from commit f6854bc9844ce94815b99032d508bda43ac1a0c1) Signed-off-by: Rohit Yadav --- server/src/com/cloud/resource/ResourceManagerImpl.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/server/src/com/cloud/resource/ResourceManagerImpl.java b/server/src/com/cloud/resource/ResourceManagerImpl.java index 77f60410981..5a1b8ce3f0e 100755 --- a/server/src/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/com/cloud/resource/ResourceManagerImpl.java @@ -31,6 +31,7 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; import com.cloud.vm.VirtualMachine; + import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.command.admin.cluster.AddClusterCmd; import org.apache.cloudstack.api.command.admin.cluster.DeleteClusterCmd; @@ -93,7 +94,9 @@ import com.cloud.dc.dao.HostPodDao; import com.cloud.deploy.PlannerHostReservationVO; import com.cloud.deploy.dao.PlannerHostReservationDao; import com.cloud.event.ActionEvent; +import com.cloud.event.ActionEventUtils; import com.cloud.event.EventTypes; +import com.cloud.event.EventVO; import com.cloud.exception.AgentUnavailableException; import com.cloud.exception.DiscoveryException; import com.cloud.exception.InvalidParameterValueException; @@ -1188,6 +1191,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, throw new CloudRuntimeException(err + e.getMessage()); } + ActionEventUtils.onStartedActionEvent(CallContext.current().getCallingUserId(), CallContext.current().getCallingAccountId(), EventTypes.EVENT_MAINTENANCE_PREPARE, "starting maintenance for host " + hostId, true, 0); _agentMgr.pullAgentToMaintenance(hostId); /* TODO: move below to listener */ @@ -1270,6 +1274,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, if (vos.isEmpty() && vosMigrating.isEmpty()) { resourceStateTransitTo(host, ResourceState.Event.InternalEnterMaintenance, _nodeId); hostInMaintenance = true; + ActionEventUtils.onCompletedActionEvent(CallContext.current().getCallingUserId(), CallContext.current().getCallingAccountId(), EventVO.LEVEL_INFO, EventTypes.EVENT_MAINTENANCE_PREPARE, "completed maintenance for host " + hostId, 0); } } } catch (NoTransitionException e) { From 535c037428b038d0d4b5e80e4c4a0e1c5d24d6b7 Mon Sep 17 00:00:00 2001 From: Chandan Purushothama Date: Tue, 2 Dec 2014 14:37:11 -0800 Subject: [PATCH 075/391] CLOUDSTACK-8007: Fixed the script 'test_vm_passwdenabled.py' - Template created by Admin should have public access to be used for regular User VM Deployment (cherry picked from commit 41b871b6bdfd48f673e1fdc011069dd3c7ca032b) Signed-off-by: Rohit Yadav Conflicts: test/integration/component/test_vm_passwdenabled.py --- test/integration/component/test_vm_passwdenabled.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/integration/component/test_vm_passwdenabled.py b/test/integration/component/test_vm_passwdenabled.py index 64e549b7746..77d30139d9a 100644 --- a/test/integration/component/test_vm_passwdenabled.py +++ b/test/integration/component/test_vm_passwdenabled.py @@ -221,8 +221,6 @@ class TestVMPasswordEnabled(cloudstackTestCase): cls.api_client, cls.services["template"], cls.volume.id, - account=cls.account.name, - domainid=cls.account.domainid ) # Delete the VM - No longer needed cls.virtual_machine.delete(cls.api_client, expunge=True) From 1b7a10068877c7bcd997029c5ef3ba8884d49f18 Mon Sep 17 00:00:00 2001 From: Chandan Purushothama Date: Mon, 1 Dec 2014 14:33:29 -0800 Subject: [PATCH 076/391] CLOUDSTACK-7996: Fixed the script test_tags.py - Tags and Template should belong to the User Account to test the case (cherry picked from commit b0d74ad6fc68fd7022ec6cf378736430f0908a75) Signed-off-by: Rohit Yadav --- test/integration/component/test_tags.py | 28 +++++++++++++------------ 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/test/integration/component/test_tags.py b/test/integration/component/test_tags.py index 824cf011415..2b05fbb1432 100644 --- a/test/integration/component/test_tags.py +++ b/test/integration/component/test_tags.py @@ -212,6 +212,12 @@ class TestResourceTags(cloudstackTestCase): cls.services["account"], admin=True, ) + + cls.user_api_client = cls.testClient.getUserApiClient( + UserName=cls.account.name, + DomainName=cls.account.domain + ) + # Create service offerings, disk offerings etc cls.service_offering = ServiceOffering.create( cls.api_client, @@ -943,14 +949,14 @@ class TestResourceTags(cloudstackTestCase): try: self.debug("Stopping the virtual machine: %s" % self.vm_1.name) #Stop virtual machine - self.vm_1.stop(self.apiclient) + self.vm_1.stop(self.user_api_client) except Exception as e: self.fail("Failed to stop VM: %s" % e) timeout = self.services["timeout"] while True: list_volume = Volume.list( - self.apiclient, + self.user_api_client, virtualmachineid=self.vm_1.id, type='ROOT', listall=True @@ -969,18 +975,18 @@ class TestResourceTags(cloudstackTestCase): self.vm_1.name) #Create template from volume template = Template.create( - self.apiclient, + self.user_api_client, self.services["template"], self.volume.id ) self.cleanup.append(template) self.debug("Created the template(%s). Now restarting the userVm: %s" % (template.name, self.vm_1.name)) - self.vm_1.start(self.apiclient) + self.vm_1.start(self.user_api_client) self.debug("Creating a tag for the template") tag = Tag.create( - self.apiclient, + self.user_api_client, resourceIds=template.id, resourceType='Template', tags={'OS': 'CentOS'} @@ -988,11 +994,9 @@ class TestResourceTags(cloudstackTestCase): self.debug("Tag created: %s" % tag.__dict__) tags = Tag.list( - self.apiclient, + self.user_api_client, listall=True, resourceType='Template', - account=self.account.name, - domainid=self.account.domainid, key='OS', value='CentOS' ) @@ -1008,7 +1012,7 @@ class TestResourceTags(cloudstackTestCase): ) Template.list( - self.apiclient, + self.user_api_client, templatefilter=\ self.services["template"]["templatefilter"], listall=True, @@ -1019,7 +1023,7 @@ class TestResourceTags(cloudstackTestCase): self.debug("Deleting the created tag..") try: tag.delete( - self.apiclient, + self.user_api_client, resourceIds=template.id, resourceType='Template', tags={'OS': 'CentOS'} @@ -1029,11 +1033,9 @@ class TestResourceTags(cloudstackTestCase): self.debug("Verifying if tag is actually deleted!") tags = Tag.list( - self.apiclient, + self.user_api_client, listall=True, resourceType='Template', - account=self.account.name, - domainid=self.account.domainid, key='OS', value='CentOS' ) From 3cb4358270497d4db22d19647a4c2820ed544f04 Mon Sep 17 00:00:00 2001 From: amoghvk Date: Wed, 26 Nov 2014 15:08:48 -0800 Subject: [PATCH 077/391] CLOUDSTACK-7977 Fix password generator, add guards for minimum length (cherry picked from commit 960b7bbf742bbba62cd25bc62b700c6c829e35f2) Signed-off-by: Rohit Yadav --- .../src/com/cloud/configuration/Config.java | 8 +++++++ .../ConfigurationManagerImpl.java | 5 ++++ .../com/cloud/utils/PasswordGenerator.java | 24 +++++++++++++------ .../cloud/utils/PasswordGeneratorTest.java | 7 +++--- 4 files changed, 34 insertions(+), 10 deletions(-) diff --git a/server/src/com/cloud/configuration/Config.java b/server/src/com/cloud/configuration/Config.java index 091a3e9740c..dfb5f2aa8fc 100755 --- a/server/src/com/cloud/configuration/Config.java +++ b/server/src/com/cloud/configuration/Config.java @@ -907,6 +907,14 @@ public enum Config { "0", "Default disk I/O read rate in requests per second allowed in User vm's disk.", null), + VmPasswordLength( + "Advanced", + ManagementServer.class, + Integer.class, + "vm.password.length", + "6", + "Specifies the length of a randomly generated password", + null), VmDiskThrottlingIopsWriteRate( "Advanced", ManagementServer.class, diff --git a/server/src/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/com/cloud/configuration/ConfigurationManagerImpl.java index ce63e8485c2..cf94b95353e 100755 --- a/server/src/com/cloud/configuration/ConfigurationManagerImpl.java +++ b/server/src/com/cloud/configuration/ConfigurationManagerImpl.java @@ -366,6 +366,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati configValuesForValidation.add("xenserver.heartbeat.interval"); configValuesForValidation.add("xenserver.heartbeat.timeout"); configValuesForValidation.add("incorrect.login.attempts.allowed"); + configValuesForValidation.add("vm.password.length"); } private void weightBasedParametersForValidation() { @@ -779,6 +780,10 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati if (val <= 0) { throw new InvalidParameterValueException("Please enter a positive value for the configuration parameter:" + name); } + //TODO - better validation for all password pamameters + if ("vm.password.length".equalsIgnoreCase(name) && val < 6) { + throw new InvalidParameterValueException("Please enter a value greater than 6 for the configuration parameter:" + name); + } } catch (NumberFormatException e) { s_logger.error("There was an error trying to parse the integer value for:" + name); throw new InvalidParameterValueException("There was an error trying to parse the integer value for:" + name); diff --git a/utils/src/com/cloud/utils/PasswordGenerator.java b/utils/src/com/cloud/utils/PasswordGenerator.java index 2abf071087f..0d791431b07 100644 --- a/utils/src/com/cloud/utils/PasswordGenerator.java +++ b/utils/src/com/cloud/utils/PasswordGenerator.java @@ -35,18 +35,28 @@ public class PasswordGenerator { static private char[] alphaNumeric = new char[] {'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'J', 'K', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'j', 'k', 'm', 'n', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '2', '3', '4', '5', '6', '7', '8', '9'}; + static private int minLength = 3; + public static String generateRandomPassword(int num) { Random r = new SecureRandom(); StringBuilder password = new StringBuilder(); - // Generate random 3-character string with a lowercase character, - // uppercase character, and a digit - password.append(generateLowercaseChar(r)).append(generateUppercaseChar(r)).append(generateDigit(r)); + //Guard for num < minLength + if (num < minLength) { + //Add alphanumeric chars at random + for (int i = 0; i < minLength; i++) { + password.append(generateAlphaNumeric(r)); + } + } else { + // Generate random 3-character string with a lowercase character, + // uppercase character, and a digit + password.append(generateLowercaseChar(r)).append(generateUppercaseChar(r)).append(generateDigit(r)); - // Generate a random n-character string with only lowercase - // characters - for (int i = 0; i < num; i++) { - password.append(generateLowercaseChar(r)); + // Generate a random n-character string with only lowercase + // characters + for (int i = 0; i < num - 3; i++) { + password.append(generateLowercaseChar(r)); + } } return password.toString(); diff --git a/utils/test/com/cloud/utils/PasswordGeneratorTest.java b/utils/test/com/cloud/utils/PasswordGeneratorTest.java index a4e249660aa..413b86618c8 100644 --- a/utils/test/com/cloud/utils/PasswordGeneratorTest.java +++ b/utils/test/com/cloud/utils/PasswordGeneratorTest.java @@ -25,10 +25,11 @@ import org.junit.Test; public class PasswordGeneratorTest { @Test public void generateRandomPassword() { - // actual length is requested length + 3 + // actual length is requested length, minimum length is 3 Assert.assertTrue(PasswordGenerator.generateRandomPassword(0).length() == 3); - Assert.assertTrue(PasswordGenerator.generateRandomPassword(1).length() == 4); - String password = PasswordGenerator.generateRandomPassword(0); + Assert.assertTrue(PasswordGenerator.generateRandomPassword(1).length() == 3); + Assert.assertTrue(PasswordGenerator.generateRandomPassword(5).length() == 5); + String password = PasswordGenerator.generateRandomPassword(8); // TODO: this might give more help to bruteforcing than desired // the actual behavior is that the first character is a random lowercase // char From 57f5d51a11e3b147432f88bcd2ba144bb9e6cdf7 Mon Sep 17 00:00:00 2001 From: amoghvk Date: Wed, 26 Nov 2014 14:54:42 -0800 Subject: [PATCH 078/391] CLOUDSTACK-7976 : Param validation for global params involving domain name (cherry picked from commit 95ea20390739a24dad92895b8db712282be31bbb) Signed-off-by: Rohit Yadav Conflicts: server/src/com/cloud/configuration/ConfigurationManagerImpl.java --- .../cloud/configuration/ConfigurationManagerImpl.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/server/src/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/com/cloud/configuration/ConfigurationManagerImpl.java index cf94b95353e..d9ce3fd3689 100755 --- a/server/src/com/cloud/configuration/ConfigurationManagerImpl.java +++ b/server/src/com/cloud/configuration/ConfigurationManagerImpl.java @@ -327,6 +327,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati private int _maxVolumeSizeInGb = Integer.parseInt(Config.MaxVolumeSize.getDefaultValue()); private long _defaultPageSize = Long.parseLong(Config.DefaultPageSize.getDefaultValue()); + private static final String DOMAIN_NAME_PATTERN = "^((?!-)[A-Za-z0-9-]{1,63}(? configValuesForValidation; private Set weightBasedParametersForValidation; private Set overprovisioningFactorsForValidation; @@ -843,6 +844,15 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati if (!NetUtils.verifyInstanceName(value)) { return "Instance name can not contain hyphen, space or plus sign"; } + } else if (range.equalsIgnoreCase("domainName")) { + String domainName = value; + if (value.startsWith("*")) { + domainName = value.substring(2); //skip the "*." + } + //max length for FQDN is 253 + 2, code adds xxx-xxx-xxx-xxx to domain name when creating URL + if (domainName.length() >= 238 || !domainName.matches(DOMAIN_NAME_PATTERN)) { + return "Please enter a valid string for domain name, prefixed with '*.' if applicable"; + } } else if (range.equals("routes")) { String[] routes = value.split(","); for (String route : routes) { From 30598e958bf50b9c5d3742a6f36553b1f825c598 Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Tue, 20 Jan 2015 11:21:10 +0530 Subject: [PATCH 079/391] setup/db: Fix database blunders - Make schema-442to450.sql same as on master branch - Make schema-430to440.sql same as on 4.4 branch Signed-off-by: Rohit Yadav --- setup/db/db/schema-430to440.sql | 7 ------- setup/db/db/schema-442to450.sql | 19 +++++++++++++++++-- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/setup/db/db/schema-430to440.sql b/setup/db/db/schema-430to440.sql index 0943518a985..ec10a3a7036 100644 --- a/setup/db/db/schema-430to440.sql +++ b/setup/db/db/schema-430to440.sql @@ -686,13 +686,6 @@ INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (hypervisor_type, guest_os_name INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (hypervisor_type, guest_os_name, guest_os_id) VALUES ("VmWare", 'CentOS 6.5 (64-bit)', 228); INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (hypervisor_type, guest_os_name, guest_os_id) VALUES ("XenServer", 'CentOS 6.5 (64-bit)', 228); -CREATE TABLE `cloud`.`baremetal_rct` ( - `id` bigint unsigned UNIQUE AUTO_INCREMENT, - `uuid` varchar(40) UNIQUE NOT NULL, - `url` varchar(2048) NOT NULL, - `rct` text NOT NULL, - PRIMARY KEY (`id`) -) ENGINE = InnoDB DEFAULT CHARSET=utf8; CREATE TABLE `cloud`.`op_router_monitoring_services` ( `vm_id` bigint unsigned UNIQUE NOT NULL COMMENT 'Primary Key', diff --git a/setup/db/db/schema-442to450.sql b/setup/db/db/schema-442to450.sql index b268eca1df4..94984816fe4 100644 --- a/setup/db/db/schema-442to450.sql +++ b/setup/db/db/schema-442to450.sql @@ -770,6 +770,8 @@ INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, crea INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (247, UUID(), 3, 'Oracle Linux 7', utc_timestamp()); INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (248, UUID(), 1, 'CentOS 6 (32-bit)', utc_timestamp()); INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (249, UUID(), 1, 'CentOS 6 (64-bit)', utc_timestamp()); +INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (250, UUID(), 3, 'Oracle Enterprise Linux 6.5 (32-bit)', utc_timestamp()); +INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (251, UUID(), 3, 'Oracle Enterprise Linux 6.5 (64-bit)', utc_timestamp()); INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'Xenserver', '6.5.0', 'CentOS 4.5 (32-bit)', 1, utc_timestamp(), 0); INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'Xenserver', '6.5.0', 'CentOS 4.6 (32-bit)', 2, utc_timestamp(), 0); @@ -852,6 +854,8 @@ INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervis INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'Xenserver', '6.5.0', 'Oracle Enterprise Linux 6 (64-bit)', 220, utc_timestamp(), 0); INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'Xenserver', '6.5.0', 'Oracle Enterprise Linux 6 (32-bit)', 235, utc_timestamp(), 0); INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'Xenserver', '6.5.0', 'Oracle Enterprise Linux 6 (64-bit)', 236, utc_timestamp(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'Xenserver', '6.5.0', 'Oracle Enterprise Linux 6 (32-bit)', 250, utc_timestamp(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'Xenserver', '6.5.0', 'Oracle Enterprise Linux 6 (64-bit)', 251, utc_timestamp(), 0); INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'Xenserver', '6.5.0', 'Oracle Linux 7', 247, utc_timestamp(), 0); INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'Xenserver', '6.5.0', 'Red Hat Enterprise Linux 4.5 (32-bit)', 26, utc_timestamp(), 0); INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'Xenserver', '6.5.0', 'Red Hat Enterprise Linux 4.6 (32-bit)', 27, utc_timestamp(), 0); @@ -948,6 +952,16 @@ INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervis INSERT IGNORE INTO `cloud`.`hypervisor_capabilities`(uuid, hypervisor_type, hypervisor_version, max_guests_limit, security_group_enabled, max_data_volumes_limit, storage_motion_supported) VALUES (UUID(), 'XenServer', '6.5.0', 500, 1, 13, 1); +update vlan set vlan_id=concat('vlan://', vlan_id) where vlan_type = "VirtualNetwork" and vlan_id not like "vlan://%"; + +CREATE TABLE `cloud`.`baremetal_rct` ( + `id` bigint unsigned UNIQUE AUTO_INCREMENT, + `uuid` varchar(40) UNIQUE NOT NULL, + `url` varchar(2048) NOT NULL, + `rct` text NOT NULL, + PRIMARY KEY (`id`) +) ENGINE = InnoDB DEFAULT CHARSET=utf8; + --Remove duplicates from guest_os_hypervisor table DELETE t1 FROM guest_os_hypervisor t1, guest_os_hypervisor t2 WHERE (t1.hypervisor_type = t2.hypervisor_type AND t1.hypervisor_version = t2.hypervisor_version AND t1.guest_os_id = t2.guest_os_id AND t1.id > t2.id AND t1.is_user_defined=0); @@ -961,6 +975,9 @@ ALTER TABLE `cloud`.`user_vm_details` MODIFY `value` VARCHAR(5120); UPDATE `cloud`.`host` SET resource = REPLACE(resource, 'com.cloud.hypervisor.xen.resource', 'com.cloud.hypervisor.xenserver.resource') WHERE hypervisor_type='XenServer' AND REMOVED IS NULL; +INSERT INTO `cloud`.`vm_template` (id, uuid, unique_name, name, public, created, type, hvm, bits, account_id, url, checksum, enable_password, display_text, format, guest_os_id, featured, cross_zones, hypervisor_type, extractable, state) + VALUES (11, UUID(), 'centos7-x86_64-lxc', 'CentOS 7(64-bit) no GUI (LXC)', 1, now(), 'BUILTIN', 0, 64, 1, 'http://download.cloud.com/templates/builtin/centos-7-x86_64.tar.gz', 'c2c4fa2d0978121c7977db571f132d6e', 0, 'CentOS 7(64-bit) no GUI (LXC)', 'TAR', 246, 1, 1, 'LXC', 1, 'Active'); + --Support for RHEL 6.5 in relevant hypervisor versions INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (252, UUID(), 4, 'Red Hat Enterprise Linux 6.5 (32-bit)', utc_timestamp()); INSERT IGNORE INTO `cloud`.`guest_os` (id, uuid, category_id, display_name, created) VALUES (253, UUID(), 4, 'Red Hat Enterprise Linux 6.5 (64-bit)', utc_timestamp()); @@ -979,8 +996,6 @@ INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervis INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.1', 'rhel6_64Guest', 253, utc_timestamp(), 0); INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5.5', 'rhel6Guest', 252, utc_timestamp(), 0); INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'VMware', '5,5', 'rhel6_64Guest', 253, utc_timestamp(), 0); -INSERT INTO `cloud`.`vm_template` (id, uuid, unique_name, name, public, created, type, hvm, bits, account_id, url, checksum, enable_password, display_text, format, guest_os_id, featured, cross_zones, hypervisor_type, extractable, state) - VALUES (11, UUID(), 'centos7-x86_64-lxc', 'CentOS 7(64-bit) no GUI (LXC)', 1, now(), 'BUILTIN', 0, 64, 1, 'http://download.cloud.com/templates/builtin/centos-7-x86_64.tar.gz', 'c2c4fa2d0978121c7977db571f132d6e', 0, 'CentOS 7(64-bit) no GUI (LXC)', 'TAR', 246, 1, 1, 'LXC', 1, 'Active'); --Support for Debian 7 on KVM/LXC INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'KVM', 'default', 'Debian GNU/Linux 7(32-bit)', 183, utc_timestamp(), 0); From bc4dca71c871fa5e57794b9552a18233b7f114fc Mon Sep 17 00:00:00 2001 From: Sanjay Tripathi Date: Wed, 26 Nov 2014 13:28:25 +0530 Subject: [PATCH 080/391] CLOUDSTACK-7969: SC: Win8.1: Key translation fails for some EN-US keyboard keys. (cherry picked from commit a45ddb514c13b0b72d47468fb2254e69520d37f6) Signed-off-by: Rohit Yadav --- ui/index.jsp | 1 + 1 file changed, 1 insertion(+) diff --git a/ui/index.jsp b/ui/index.jsp index 83d5d54987f..fc2a742e2af 100644 --- a/ui/index.jsp +++ b/ui/index.jsp @@ -440,6 +440,7 @@ + From b77cbea0f35b2bcdbbd41f742032207c4aefbf1b Mon Sep 17 00:00:00 2001 From: Sanjay Tripathi Date: Mon, 24 Nov 2014 13:37:18 +0530 Subject: [PATCH 081/391] CLOUDSTACK-7964: listAccounts API is not listing correct value of resource limits. (cherry picked from commit d475b62838878677531d6daab667757baf63604e) Signed-off-by: Rohit Yadav --- server/src/com/cloud/api/query/dao/AccountJoinDaoImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/com/cloud/api/query/dao/AccountJoinDaoImpl.java b/server/src/com/cloud/api/query/dao/AccountJoinDaoImpl.java index 31d5a2c5e82..fadaed52874 100755 --- a/server/src/com/cloud/api/query/dao/AccountJoinDaoImpl.java +++ b/server/src/com/cloud/api/query/dao/AccountJoinDaoImpl.java @@ -75,7 +75,7 @@ public class AccountJoinDaoImpl extends GenericDaoBase impl accountResponse.setBytesReceived(account.getBytesReceived()); accountResponse.setBytesSent(account.getBytesSent()); - boolean fullView = (view == ResponseView.Full); + boolean fullView = (view == ResponseView.Full && _acctMgr.isRootAdmin(account.getId())); setResourceLimits(account, fullView, accountResponse); //get resource limits for projects From f88a34dabe1f3a18cd508e31fdc1da8077866a27 Mon Sep 17 00:00:00 2001 From: Saksham Srivastava Date: Fri, 21 Nov 2014 17:28:06 +0530 Subject: [PATCH 082/391] CLOUDSTACK-7962: VM Snaptshot should add check for root volume status also (cherry picked from commit 690a5ded358fe6f4d8671f497049f510d4894b1e) Signed-off-by: Rohit Yadav --- .../cloud/vm/snapshot/VMSnapshotManagerImpl.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/server/src/com/cloud/vm/snapshot/VMSnapshotManagerImpl.java b/server/src/com/cloud/vm/snapshot/VMSnapshotManagerImpl.java index e7df0886ccb..02687278eca 100644 --- a/server/src/com/cloud/vm/snapshot/VMSnapshotManagerImpl.java +++ b/server/src/com/cloud/vm/snapshot/VMSnapshotManagerImpl.java @@ -29,7 +29,6 @@ import javax.naming.ConfigurationException; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; - import org.apache.cloudstack.api.command.user.vmsnapshot.ListVMSnapshotCmd; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.subsystem.api.storage.StorageStrategyFactory; @@ -62,6 +61,7 @@ import com.cloud.projects.Project.ListProjectResourcesCriteria; import com.cloud.service.dao.ServiceOfferingDetailsDao; import com.cloud.storage.Snapshot; import com.cloud.storage.SnapshotVO; +import com.cloud.storage.Volume.Type; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.GuestOSDao; import com.cloud.storage.dao.SnapshotDao; @@ -413,6 +413,17 @@ public class VMSnapshotManagerImpl extends ManagerBase implements VMSnapshotMana if (userVm == null) { throw new InvalidParameterValueException("Create vm to snapshot failed due to vm: " + vmId + " is not found"); } + + List volumeVos = _volumeDao.findByInstanceAndType(vmId, Type.ROOT); + if(volumeVos == null ||volumeVos.isEmpty()) { + throw new CloudRuntimeException("Create vm to snapshot failed due to no root disk found"); + } + + VolumeVO rootVolume = volumeVos.get(0); + if(!rootVolume.getState().equals(State.Running)) { + throw new CloudRuntimeException("Create vm to snapshot failed due to vm: " + vmId + " has root disk in " + rootVolume.getState() + " state"); + } + VMSnapshotVO vmSnapshot = _vmSnapshotDao.findById(vmSnapshotId); if (vmSnapshot == null) { throw new CloudRuntimeException("VM snapshot id: " + vmSnapshotId + " can not be found"); From 6e5b9da7d48919496929a8da2f7f1ebd897156f7 Mon Sep 17 00:00:00 2001 From: Saksham Srivastava Date: Mon, 24 Nov 2014 12:08:58 +0530 Subject: [PATCH 083/391] CLOUDSTACK-7962: Change state to Volume State (cherry picked from commit c615aafd7706d3036e5bd1e95671b3aa0c287cea) Signed-off-by: Rohit Yadav --- server/src/com/cloud/vm/snapshot/VMSnapshotManagerImpl.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/com/cloud/vm/snapshot/VMSnapshotManagerImpl.java b/server/src/com/cloud/vm/snapshot/VMSnapshotManagerImpl.java index 02687278eca..b49db9b51c1 100644 --- a/server/src/com/cloud/vm/snapshot/VMSnapshotManagerImpl.java +++ b/server/src/com/cloud/vm/snapshot/VMSnapshotManagerImpl.java @@ -61,6 +61,7 @@ import com.cloud.projects.Project.ListProjectResourcesCriteria; import com.cloud.service.dao.ServiceOfferingDetailsDao; import com.cloud.storage.Snapshot; import com.cloud.storage.SnapshotVO; +import com.cloud.storage.Volume; import com.cloud.storage.Volume.Type; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.GuestOSDao; @@ -420,7 +421,7 @@ public class VMSnapshotManagerImpl extends ManagerBase implements VMSnapshotMana } VolumeVO rootVolume = volumeVos.get(0); - if(!rootVolume.getState().equals(State.Running)) { + if(!rootVolume.getState().equals(Volume.State.Ready)) { throw new CloudRuntimeException("Create vm to snapshot failed due to vm: " + vmId + " has root disk in " + rootVolume.getState() + " state"); } From d1d288522de08fe014e91ed8744e38c96e3b0ed1 Mon Sep 17 00:00:00 2001 From: Chandan Purushothama Date: Thu, 20 Nov 2014 13:45:35 -0800 Subject: [PATCH 084/391] CLOUDSTACK-7956: Fixed the script 'test_project_usage.py' - Register Template in the Project to test the Template limits on the project (cherry picked from commit 3a6f248e3af0b5d0ba3af7748fc650c44a280ec7) Signed-off-by: Rohit Yadav --- .../component/test_project_usage.py | 79 ++++++++++++++++--- 1 file changed, 67 insertions(+), 12 deletions(-) diff --git a/test/integration/component/test_project_usage.py b/test/integration/component/test_project_usage.py index 26275045e71..29d381ab187 100644 --- a/test/integration/component/test_project_usage.py +++ b/test/integration/component/test_project_usage.py @@ -39,7 +39,9 @@ from marvin.lib.base import (Project, from marvin.lib.common import (get_domain, get_zone, get_template, - list_volumes) + list_volumes, + get_builtin_template_info) +import time class Services: """Test Snapshots Services @@ -784,23 +786,76 @@ class TestTemplateUsage(cloudstackTestCase): """ # Validate the following # 1. Create a account - # 2. Upload a template from this account. template.create event is + # 2. Register template in the project. template.create event is # recorded in cloud.usage_event table for this account # 3. Delete the template. template.delete event is recorded in # cloud.usage_event tables for this account # 4. Destroy the account - #Create template from Virtual machine and Volume ID - self.template = Template.create( - self.userapiclient, - self.services["templates"], - self.volume.id, - projectid=self.project.id - ) - self.debug("Created template with ID: %s" % self.template.id) + # Register the First Template in the project + self.debug("Register a Template in the project") + builtin_info = get_builtin_template_info(self.apiclient, self.zone.id) + self.services["templates"]["url"] = builtin_info[0] + self.services["templates"]["hypervisor"] = builtin_info[1] + self.services["templates"]["format"] = builtin_info[2] + + # Register new template + template = Template.register( + self.userapiclient, + self.services["templates"], + zoneid=self.zone.id, + projectid=self.project.id + ) + self.debug( + "Registered a template of format: %s with ID: %s" % ( + self.services["templates"]["format"], + template.id + )) + + # Wait for template status to be changed across + time.sleep(self.services["sleep"]) + timeout = self.services["timeout"] + while True: + list_template_response = Template.list( + self.apiclient, + templatefilter='all', + id=template.id, + zoneid=self.zone.id, + projectid=self.project.id, + ) + if list_template_response[0].isready is True: + break + elif timeout == 0: + raise Exception("Template state is not ready, it is %s" % list_template_response[0].isready) + + time.sleep(self.services["sleep"]) + timeout = timeout - 1 + + #Verify template response to check whether template added successfully + self.assertEqual( + isinstance(list_template_response, list), + True, + "Check for list template response return valid data" + ) + + self.assertNotEqual( + len(list_template_response), + 0, + "Check template available in List Templates" + ) + + template_response = list_template_response[0] + self.assertEqual( + template_response.isready, + True, + "Template state is not ready, it is %s" % template_response.isready + ) + + self.debug("Created template with ID: %s" % template.id) + # Delete template - self.template.delete(self.apiclient) - self.debug("Deleted template with ID: %s" % self.template.id) + template.delete(self.apiclient) + self.debug("Deleted template with ID: %s" % template.id) # Fetch project account ID from project UUID self.debug( From b5ef7de29096bf222685ab29e3e4ae18083eba58 Mon Sep 17 00:00:00 2001 From: Chandan Purushothama Date: Thu, 20 Nov 2014 11:03:23 -0800 Subject: [PATCH 085/391] CLOUDSTACK-7955: Fixed the script test_project_limits.py - Register Template in the Project to test the Template limits on the project (cherry picked from commit 71bb436c8d2500cccef49e8a503663b79ef1fd71) Signed-off-by: Rohit Yadav --- .../component/test_project_limits.py | 105 ++++++++++-------- 1 file changed, 59 insertions(+), 46 deletions(-) diff --git a/test/integration/component/test_project_limits.py b/test/integration/component/test_project_limits.py index 5d37f0b3450..d4fb92d8da9 100644 --- a/test/integration/component/test_project_limits.py +++ b/test/integration/component/test_project_limits.py @@ -42,7 +42,8 @@ from marvin.lib.common import (get_domain, list_volumes, list_configurations, list_resource_limits, - update_resource_limit + update_resource_limit, + get_builtin_template_info ) from marvin.codes import PASS import time @@ -109,7 +110,7 @@ class Services: "ostype": 'CentOS 5.3 (64-bit)', "templatefilter": 'self', }, - "network_offering": { + "network_offering": { "name": 'Network offering-VR services', "displaytext": 'Network offering-VR services', "guestiptype": 'Isolated', @@ -904,65 +905,77 @@ class TestResourceLimitsProject(cloudstackTestCase): max=1, projectid=self.project.id ) + + # Register the First Template in the project + self.debug("Register the First Template in the project") + builtin_info = get_builtin_template_info(self.apiclient, self.zone.id) + self.services["template"]["url"] = builtin_info[0] + self.services["template"]["hypervisor"] = builtin_info[1] + self.services["template"]["format"] = builtin_info[2] - self.debug("Deploying VM for account: %s" % self.account.name) - virtual_machine_1 = VirtualMachine.create( - self.apiclient, - self.services["server"], - templateid=self.template.id, - serviceofferingid=self.service_offering.id, - projectid=self.project.id - ) - self.cleanup.append(virtual_machine_1) - # Verify VM state + # Register new template + template = Template.register( + self.userapiclient, + self.services["template"], + zoneid=self.zone.id, + projectid=self.project.id + ) + self.debug( + "Registered a template of format: %s with ID: %s" % ( + self.services["template"]["format"], + template.id + )) + self.cleanup.append(template) + + # Wait for template status to be changed across + time.sleep(self.services["sleep"]) + timeout = self.services["timeout"] + while True: + list_template_response = Template.list( + self.apiclient, + templatefilter='all', + id=template.id, + zoneid=self.zone.id, + projectid=self.project.id, + ) + if list_template_response[0].isready is True: + break + elif timeout == 0: + raise Exception("Template state is not ready, it is %s" % list_template_response[0].isready) + + time.sleep(self.services["sleep"]) + timeout = timeout - 1 + + #Verify template response to check whether template added successfully self.assertEqual( - virtual_machine_1.state, - 'Running', - "Check VM state is Running or not" - ) - virtual_machine_1.stop(self.apiclient) - # Get the Root disk of VM - volumes = list_volumes( - self.apiclient, - virtualmachineid=virtual_machine_1.id, - projectid=self.project.id, - type='ROOT' - ) - self.assertEqual( - isinstance(volumes, list), + isinstance(list_template_response, list), True, - "Check for list volume response return valid data" + "Check for list template response return valid data" ) - volume = volumes[0] - self.debug("Creating template from volume: %s" % volume.id) - # Create a template from the ROOTDISK - template_1 = Template.create( - self.userapiclient, - self.services["template"], - volumeid=volume.id, - projectid=self.project.id - ) + self.assertNotEqual( + len(list_template_response), + 0, + "Check template available in List Templates" + ) - self.cleanup.append(template_1) - # Verify Template state + template_response = list_template_response[0] self.assertEqual( - template_1.isready, + template_response.isready, True, - "Check Template is in ready state or not" + "Template state is not ready, it is %s" % template_response.isready ) # Exception should be raised for second template with self.assertRaises(Exception): - Template.create( - self.userapiclient, - self.services["template"], - volumeid=volume.id, - projectid=self.project.id + Template.register( + self.userapiclient, + self.services["template"], + zoneid=self.zone.id, + projectid=self.project.id ) return - class TestMaxProjectNetworks(cloudstackTestCase): @classmethod From dca38d3fa87b99b603f98be3ee38a702efc7ed28 Mon Sep 17 00:00:00 2001 From: Saksham Srivastava Date: Tue, 18 Nov 2014 15:16:37 +0530 Subject: [PATCH 086/391] CLOUDSTACK-7950: AttachIsoCmd shoud give correct messge when trying to attach vmwaretools installer iso on non supported guestvm deployed by ISO (cherry picked from commit 4ff3130becd50ab8d44864b651c1b1e235d67e6f) Signed-off-by: Rohit Yadav --- .../src/com/cloud/storage/resource/VmwareStorageProcessor.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageProcessor.java index 19336490026..757826154d1 100644 --- a/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -1452,10 +1452,12 @@ public class VmwareStorageProcessor implements StorageProcessor { if (isAttach) { String msg = "AttachIsoCommand(attach) failed due to " + VmwareHelper.getExceptionMessage(e); + msg = msg + " Also check if your guest os is a supported version"; s_logger.error(msg, e); return new AttachAnswer(msg); } else { String msg = "AttachIsoCommand(detach) failed due to " + VmwareHelper.getExceptionMessage(e); + msg = msg + " Also check if your guest os is a supported version"; s_logger.warn(msg, e); return new AttachAnswer(msg); } From 6715c6ccfa68b53a3891f49ec98b568511c9cfb9 Mon Sep 17 00:00:00 2001 From: Saksham Srivastava Date: Mon, 17 Nov 2014 16:28:49 +0530 Subject: [PATCH 087/391] CLOUDSTACK-7941: CloudStack should log IP address of actual client even if a ReverseProxy is there (cherry picked from commit f0a4a639de231929bd63f673c4d6adc6bfb5ca80) Signed-off-by: Rohit Yadav --- server/src/com/cloud/api/ApiServlet.java | 96 ++++++++++++++++++------ 1 file changed, 74 insertions(+), 22 deletions(-) diff --git a/server/src/com/cloud/api/ApiServlet.java b/server/src/com/cloud/api/ApiServlet.java index c7d06093a1a..3d2e843db30 100644 --- a/server/src/com/cloud/api/ApiServlet.java +++ b/server/src/com/cloud/api/ApiServlet.java @@ -16,23 +16,10 @@ // under the License. package com.cloud.api; -import org.apache.cloudstack.api.auth.APIAuthenticationManager; -import org.apache.cloudstack.api.auth.APIAuthenticationType; -import org.apache.cloudstack.api.auth.APIAuthenticator; -import com.cloud.user.Account; -import com.cloud.user.AccountService; -import com.cloud.user.User; -import com.cloud.utils.HttpUtils; -import com.cloud.utils.StringUtils; -import com.cloud.utils.db.EntityManager; -import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.ApiServerService; -import org.apache.cloudstack.api.ServerApiException; -import org.apache.cloudstack.context.CallContext; -import org.apache.cloudstack.managed.context.ManagedContext; -import org.apache.log4j.Logger; -import org.springframework.stereotype.Component; -import org.springframework.web.context.support.SpringBeanAutowiringSupport; +import java.io.UnsupportedEncodingException; +import java.net.URLDecoder; +import java.util.HashMap; +import java.util.Map; import javax.inject.Inject; import javax.servlet.ServletConfig; @@ -41,10 +28,26 @@ import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpSession; -import java.io.UnsupportedEncodingException; -import java.net.URLDecoder; -import java.util.HashMap; -import java.util.Map; + +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiServerService; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.auth.APIAuthenticationManager; +import org.apache.cloudstack.api.auth.APIAuthenticationType; +import org.apache.cloudstack.api.auth.APIAuthenticator; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.managed.context.ManagedContext; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; +import org.springframework.web.context.support.SpringBeanAutowiringSupport; + +import com.cloud.user.Account; +import com.cloud.user.AccountService; +import com.cloud.user.User; +import com.cloud.utils.HttpUtils; +import com.cloud.utils.StringUtils; +import com.cloud.utils.db.EntityManager; +import com.cloud.utils.net.NetUtils; @Component("apiServlet") @SuppressWarnings("serial") @@ -120,7 +123,7 @@ public class ApiServlet extends HttpServlet { } void processRequestInContext(final HttpServletRequest req, final HttpServletResponse resp) { - final String remoteAddress = req.getRemoteAddr(); + final String remoteAddress = getClientAddress(req); final StringBuilder auditTrailSb = new StringBuilder(128); auditTrailSb.append(" ").append(remoteAddress); auditTrailSb.append(" -- ").append(req.getMethod()).append(' '); @@ -304,4 +307,53 @@ public class ApiServlet extends HttpServlet { CallContext.unregister(); } } + + //This method will try to get login IP of user even if servlet is behind reverseProxy or loadBalancer + private String getClientAddress(HttpServletRequest request) { + String ip = null; + ip = request.getHeader("X-Forwarded-For"); + ip = getCorrectIPAddress(ip); + if (ip != null) { + return ip; + } + + ip = request.getHeader("HTTP_CLIENT_IP"); + ip = getCorrectIPAddress(ip); + if (ip != null) { + return ip; + } + + ip = request.getHeader("HTTP_X_FORWARDED_FOR"); + ip = getCorrectIPAddress(ip); + if (ip != null) { + return ip; + } + + ip = request.getHeader("Remote_Addr"); + ip = getCorrectIPAddress(ip); + if (ip != null) { + return ip; + } + + ip = request.getRemoteAddr(); + return ip; + } + + private String getCorrectIPAddress(String ip) { + if(ip == null || ip.length() == 0 || "unknown".equalsIgnoreCase(ip)) { + return null; + } + if(NetUtils.isValidIp(ip)) { + return ip; + } + //it could be possible to have multiple IPs in HTTP header, this happens if there are multiple proxy in between + //the client and the servlet, so parse the client IP + String[] ips = ip.split(","); + for(String i : ips) { + if(NetUtils.isValidIp(i.trim())) { + return i.trim(); + } + } + return null; + } } From 8790b84b20dcf347459b5a69cb338a2f65af1414 Mon Sep 17 00:00:00 2001 From: Sanjay Tripathi Date: Wed, 19 Nov 2014 15:57:13 +0530 Subject: [PATCH 088/391] CLOUDSTACK-7940: Exception printed completely on the UI. Not in a readable format. (cherry picked from commit dda29949366f16c06445dde9161c3b1151ed3f36) Signed-off-by: Rohit Yadav --- .../src/com/cloud/vm/VirtualMachineManagerImpl.java | 2 +- .../engine/orchestration/VolumeOrchestrator.java | 2 +- .../hypervisor/xenserver/resource/CitrixResourceBase.java | 7 +++---- .../storage/motion/XenServerStorageMotionStrategy.java | 6 +++--- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java index b992ed95125..0051cff843f 100755 --- a/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java @@ -1919,7 +1919,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac Answer ma = _agentMgr.send(vm.getLastHostId(), mc); if (ma == null || !ma.getResult()) { String details = (ma != null) ? ma.getDetails() : "null answer returned"; - throw new CloudRuntimeException("Unable to migrate due to " + details); + throw new CloudRuntimeException(details); } } catch (OperationTimedoutException e) { if (e.isActive()) { diff --git a/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java index 0895e84c551..fd4883b1f7f 100644 --- a/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java +++ b/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java @@ -988,7 +988,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati CommandResult result = future.get(); if (result.isFailed()) { s_logger.debug("Failed to migrated vm " + vm + " along with its volumes. " + result.getResult()); - throw new CloudRuntimeException("Failed to migrated vm " + vm + " along with its volumes. " + result.getResult()); + throw new CloudRuntimeException("Failed to migrated vm " + vm + " along with its volumes. "); } } catch (InterruptedException e) { s_logger.debug("Failed to migrated vm " + vm + " along with its volumes.", e); diff --git a/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java b/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java index b4e5cbc2467..ee4e3d534de 100644 --- a/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java +++ b/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java @@ -2956,9 +2956,8 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe } return new MigrateAnswer(cmd, true, "migration succeeded", null); } catch (Exception e) { - String msg = "Catch Exception " + e.getClass().getName() + ": Migration failed due to " + e.toString(); - s_logger.warn(msg, e); - return new MigrateAnswer(cmd, false, msg, null); + s_logger.warn(e.getMessage(), e); + return new MigrateAnswer(cmd, false, e.getMessage(), null); } } @@ -3363,7 +3362,7 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe throw new CloudRuntimeException("migrate VM catch HandleInvalid and VM is not running on dest host"); } } catch (XenAPIException e) { - String msg = "Unable to migrate VM(" + vmName + ") from host(" + _host.uuid + ") due to " + e.toString(); + String msg = "Unable to migrate VM(" + vmName + ") from host(" + _host.uuid + ")"; s_logger.warn(msg, e); throw new CloudRuntimeException(msg); } finally { diff --git a/plugins/hypervisors/xenserver/src/org/apache/cloudstack/storage/motion/XenServerStorageMotionStrategy.java b/plugins/hypervisors/xenserver/src/org/apache/cloudstack/storage/motion/XenServerStorageMotionStrategy.java index 975deec531f..c8367dd3ac1 100644 --- a/plugins/hypervisors/xenserver/src/org/apache/cloudstack/storage/motion/XenServerStorageMotionStrategy.java +++ b/plugins/hypervisors/xenserver/src/org/apache/cloudstack/storage/motion/XenServerStorageMotionStrategy.java @@ -154,7 +154,7 @@ public class XenServerStorageMotionStrategy implements DataMotionStrategy { throw new CloudRuntimeException("Error while migrating the vm " + vm + " to host " + destHost); } else if (!receiveAnswer.getResult()) { s_logger.error("Migration with storage of vm " + vm + " failed. Details: " + receiveAnswer.getDetails()); - throw new CloudRuntimeException("Error while migrating the vm " + vm + " to host " + destHost + ". " + receiveAnswer.getDetails()); + throw new CloudRuntimeException("Error while migrating the vm " + vm + " to host " + destHost); } MigrateWithStorageSendCommand sendCmd = @@ -165,7 +165,7 @@ public class XenServerStorageMotionStrategy implements DataMotionStrategy { throw new CloudRuntimeException("Error while migrating the vm " + vm + " to host " + destHost); } else if (!sendAnswer.getResult()) { s_logger.error("Migration with storage of vm " + vm + " failed. Details: " + sendAnswer.getDetails()); - throw new CloudRuntimeException("Error while migrating the vm " + vm + " to host " + destHost + ". " + sendAnswer.getDetails()); + throw new CloudRuntimeException("Error while migrating the vm " + vm + " to host " + destHost); } MigrateWithStorageCompleteCommand command = new MigrateWithStorageCompleteCommand(to); @@ -175,7 +175,7 @@ public class XenServerStorageMotionStrategy implements DataMotionStrategy { throw new CloudRuntimeException("Error while migrating the vm " + vm + " to host " + destHost); } else if (!answer.getResult()) { s_logger.error("Migration with storage of vm " + vm + " failed. Details: " + answer.getDetails()); - throw new CloudRuntimeException("Error while migrating the vm " + vm + " to host " + destHost + ". " + answer.getDetails()); + throw new CloudRuntimeException("Error while migrating the vm " + vm + " to host " + destHost); } else { // Update the volume details after migration. updateVolumePathsAfterMigration(volumeToPool, answer.getVolumeTos()); From 1aeaf4dcc9b0c34f6ed98f90dc7d059bf9cf05e2 Mon Sep 17 00:00:00 2001 From: Nitin Mehta Date: Fri, 14 Nov 2014 16:19:10 -0800 Subject: [PATCH 089/391] CLOUDSTACK-7920: NPE in the payload was causing the ssvm agent to not connect, fix it and also make sure that template/volume sync are robust that exceptions do not cause ssvm agent disconnect issues. (cherry picked from commit 5213401ace08f4a5247a325e9d99a4a66bd7df11) Signed-off-by: Rohit Yadav Conflicts: engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java --- .../storage/datastore/db/VolumeDataStoreDao.java | 2 ++ .../storage/image/db/VolumeDataStoreDaoImpl.java | 16 ++++++++++++++++ .../storage/volume/VolumeServiceImpl.java | 3 ++- .../cloud/storage/download/DownloadListener.java | 12 ++++++++---- 4 files changed, 28 insertions(+), 5 deletions(-) diff --git a/engine/schema/src/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreDao.java b/engine/schema/src/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreDao.java index 625cc297274..a0561521a93 100644 --- a/engine/schema/src/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreDao.java +++ b/engine/schema/src/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreDao.java @@ -46,4 +46,6 @@ public interface VolumeDataStoreDao extends GenericDao, List listVolumeDownloadUrls(); void expireDnldUrlsForZone(Long dcId); + + List listUploadedVolumesByStoreId(long id); } diff --git a/engine/storage/src/org/apache/cloudstack/storage/image/db/VolumeDataStoreDaoImpl.java b/engine/storage/src/org/apache/cloudstack/storage/image/db/VolumeDataStoreDaoImpl.java index 2e29ad79cf7..b71eb2c2324 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/image/db/VolumeDataStoreDaoImpl.java +++ b/engine/storage/src/org/apache/cloudstack/storage/image/db/VolumeDataStoreDaoImpl.java @@ -52,6 +52,7 @@ public class VolumeDataStoreDaoImpl extends GenericDaoBase cacheSearch; private SearchBuilder storeVolumeSearch; private SearchBuilder downloadVolumeSearch; + private SearchBuilder uploadVolumeSearch; private static final String EXPIRE_DOWNLOAD_URLS_FOR_ZONE = "update volume_store_ref set download_url_created=? where store_id in (select id from image_store where data_center_id=?)"; @@ -95,6 +96,12 @@ public class VolumeDataStoreDaoImpl extends GenericDaoBase listUploadedVolumesByStoreId(long id) { + SearchCriteria sc = uploadVolumeSearch.create(); + sc.setParameters("store_id", id); + sc.setParameters("destroyed", false); + return listIncludingRemovedBy(sc); + } + + @Override public void expireDnldUrlsForZone(Long dcId){ TransactionLegacy txn = TransactionLegacy.currentTxn(); diff --git a/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java b/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java index 6a66ea3888a..88f098c642e 100644 --- a/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java +++ b/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java @@ -26,6 +26,7 @@ import java.util.Map; import javax.inject.Inject; +import com.cloud.storage.RegisterVolumePayload; import org.apache.cloudstack.engine.cloud.entity.api.VolumeEntity; import org.apache.cloudstack.engine.subsystem.api.storage.ChapInfo; import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; @@ -1382,7 +1383,7 @@ public class VolumeServiceImpl implements VolumeService { return; } - List dbVolumes = _volumeStoreDao.listByStoreId(storeId); + List dbVolumes = _volumeStoreDao.listUploadedVolumesByStoreId(storeId); List toBeDownloaded = new ArrayList(dbVolumes); for (VolumeDataStoreVO volumeStore : dbVolumes) { VolumeVO volume = volDao.findById(volumeStore.getVolumeId()); diff --git a/server/src/com/cloud/storage/download/DownloadListener.java b/server/src/com/cloud/storage/download/DownloadListener.java index 4f9a5605e33..814ce3c44be 100755 --- a/server/src/com/cloud/storage/download/DownloadListener.java +++ b/server/src/com/cloud/storage/download/DownloadListener.java @@ -298,10 +298,14 @@ public class DownloadListener implements Listener { } }*/ else if (cmd instanceof StartupSecondaryStorageCommand) { - List imageStores = _storeMgr.getImageStoresByScope(new ZoneScope(agent.getDataCenterId())); - for (DataStore store : imageStores) { - _volumeSrv.handleVolumeSync(store); - _imageSrv.handleTemplateSync(store); + try{ + List imageStores = _storeMgr.getImageStoresByScope(new ZoneScope(agent.getDataCenterId())); + for (DataStore store : imageStores) { + _volumeSrv.handleVolumeSync(store); + _imageSrv.handleTemplateSync(store); + } + }catch (Exception e){ + s_logger.error("Caught exception while doing template/volume sync ", e); } } } From ff70ef797caac3cdaa71bb82c360c6b8f2ab83b9 Mon Sep 17 00:00:00 2001 From: Min Chen Date: Fri, 19 Dec 2014 16:48:45 -0800 Subject: [PATCH 090/391] CLOUDSTACK-8101: volume sync not working as expected - MS restart during upload volume leaves volume in hung state. (cherry picked from commit e559b15b6a166e2eb5f9b044338295fe8c9d219d) Signed-off-by: Rohit Yadav Conflicts: engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java --- .../apache/cloudstack/storage/volume/VolumeServiceImpl.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java b/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java index 88f098c642e..1dd2a86aaa3 100644 --- a/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java +++ b/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java @@ -1383,7 +1383,8 @@ public class VolumeServiceImpl implements VolumeService { return; } - List dbVolumes = _volumeStoreDao.listUploadedVolumesByStoreId(storeId); + // find all the db volumes including those with NULL url column to avoid accidentally deleting volumes on image store later. + List dbVolumes = _volumeStoreDao.listByStoreId(storeId); List toBeDownloaded = new ArrayList(dbVolumes); for (VolumeDataStoreVO volumeStore : dbVolumes) { VolumeVO volume = volDao.findById(volumeStore.getVolumeId()); From 4e01d61f87104ba877a8c7059ed484cda963f372 Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Tue, 20 Jan 2015 11:45:34 +0530 Subject: [PATCH 091/391] engine: remove redundant import Signed-off-by: Rohit Yadav --- .../org/apache/cloudstack/storage/volume/VolumeServiceImpl.java | 1 - 1 file changed, 1 deletion(-) diff --git a/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java b/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java index 1dd2a86aaa3..b17031c59b4 100644 --- a/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java +++ b/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java @@ -26,7 +26,6 @@ import java.util.Map; import javax.inject.Inject; -import com.cloud.storage.RegisterVolumePayload; import org.apache.cloudstack.engine.cloud.entity.api.VolumeEntity; import org.apache.cloudstack.engine.subsystem.api.storage.ChapInfo; import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; From a12ab8a2b3a14d2fed2bca287823bb558d0501fb Mon Sep 17 00:00:00 2001 From: Nitin Mehta Date: Fri, 14 Nov 2014 13:18:27 -0800 Subject: [PATCH 092/391] CLOUDSTACK-7916: Generate Alerts if System VMs cannot be started. (cherry picked from commit 3907bc6f9d2b4a61a4d0b32e5c1a15d95210531e) Signed-off-by: Rohit Yadav --- .../cloud/alert/ConsoleProxyAlertAdapter.java | 12 +- .../alert/SecondaryStorageVmAlertAdapter.java | 11 +- .../consoleproxy/ConsoleProxyManagerImpl.java | 74 +++++++----- .../SecondaryStorageManagerImpl.java | 114 ++++++++++-------- 4 files changed, 114 insertions(+), 97 deletions(-) diff --git a/server/src/com/cloud/alert/ConsoleProxyAlertAdapter.java b/server/src/com/cloud/alert/ConsoleProxyAlertAdapter.java index df4f5ba953e..4ca5f4a216b 100644 --- a/server/src/com/cloud/alert/ConsoleProxyAlertAdapter.java +++ b/server/src/com/cloud/alert/ConsoleProxyAlertAdapter.java @@ -53,7 +53,8 @@ public class ConsoleProxyAlertAdapter extends AdapterBase implements AlertAdapte DataCenterVO dc = _dcDao.findById(args.getZoneId()); ConsoleProxyVO proxy = args.getProxy(); - if (proxy == null) + //FIXME - Proxy can be null in case of creation failure. Have a better fix than checking for != 0 + if (proxy == null && args.getProxyId() != 0) proxy = _consoleProxyDao.findById(args.getProxyId()); switch (args.getType()) { @@ -98,12 +99,9 @@ public class ConsoleProxyAlertAdapter extends AdapterBase implements AlertAdapte case ConsoleProxyAlertEventArgs.PROXY_CREATE_FAILURE: if (s_logger.isDebugEnabled()) - s_logger.debug("Console proxy creation failure, zone: " + dc.getName() + ", proxy: " + proxy.getHostName() + ", public IP: " + - proxy.getPublicIpAddress() + ", private IP: " + (proxy.getPrivateIpAddress() == null ? "N/A" : proxy.getPrivateIpAddress())); - - _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_CONSOLE_PROXY, args.getZoneId(), proxy.getPodIdToDeployIn(), - "Console proxy creation failure. zone: " + dc.getName() + ", proxy: " + proxy.getHostName() + ", public IP: " + proxy.getPublicIpAddress() + - ", private IP: " + (proxy.getPrivateIpAddress() == null ? "N/A" : proxy.getPrivateIpAddress()) + ", error details: " + args.getMessage(), + s_logger.debug("Console proxy creation failure, zone: " + dc.getName()); + _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_CONSOLE_PROXY, args.getZoneId(), null, + "Console proxy creation failure. zone: " + dc.getName() + ", error details: " + args.getMessage(), "Console proxy creation failure (zone " + dc.getName() + ")"); break; diff --git a/server/src/com/cloud/alert/SecondaryStorageVmAlertAdapter.java b/server/src/com/cloud/alert/SecondaryStorageVmAlertAdapter.java index b7834bb0341..3bfedce3b0f 100644 --- a/server/src/com/cloud/alert/SecondaryStorageVmAlertAdapter.java +++ b/server/src/com/cloud/alert/SecondaryStorageVmAlertAdapter.java @@ -53,7 +53,7 @@ public class SecondaryStorageVmAlertAdapter extends AdapterBase implements Alert DataCenterVO dc = _dcDao.findById(args.getZoneId()); SecondaryStorageVmVO secStorageVm = args.getSecStorageVm(); - if (secStorageVm == null) + if (secStorageVm == null && args.getSecStorageVmId() != 0) secStorageVm = _ssvmDao.findById(args.getSecStorageVmId()); switch (args.getType()) { @@ -103,13 +103,10 @@ public class SecondaryStorageVmAlertAdapter extends AdapterBase implements Alert case SecStorageVmAlertEventArgs.SSVM_CREATE_FAILURE: if (s_logger.isDebugEnabled()) - s_logger.debug("Secondary Storage Vm creation failure, zone: " + dc.getName() + ", secStorageVm: " + secStorageVm.getHostName() + ", public IP: " + - secStorageVm.getPublicIpAddress() + ", private IP: " + (secStorageVm.getPrivateIpAddress() == null ? "N/A" : secStorageVm.getPrivateIpAddress())); + s_logger.debug("Secondary Storage Vm creation failure, zone: " + dc.getName()); - _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_SSVM, args.getZoneId(), secStorageVm.getPodIdToDeployIn(), - "Secondary Storage Vm creation failure. zone: " + - dc.getName() + ", secStorageVm: " + secStorageVm.getHostName() + ", public IP: " + secStorageVm.getPublicIpAddress() + ", private IP: " + - (secStorageVm.getPrivateIpAddress() == null ? "N/A" : secStorageVm.getPrivateIpAddress()) + ", error details: " + args.getMessage(), + _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_SSVM, args.getZoneId(), null, + "Secondary Storage Vm creation failure. zone: " + dc.getName() + ", error details: " + args.getMessage(), "Secondary Storage Vm creation failure (zone " + dc.getName() + ")"); break; diff --git a/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java b/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java index 7f2b52ab4cb..a527160e1c3 100755 --- a/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java +++ b/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java @@ -535,6 +535,7 @@ public class ConsoleProxyManagerImpl extends ManagerBase implements ConsoleProxy if (proxy.getState() == VirtualMachine.State.Stopped) { _itMgr.advanceStart(proxy.getUuid(), null, null); proxy = _consoleProxyDao.findById(proxy.getId()); + return proxy; } // For VMs that are in Stopping, Starting, Migrating state, let client to wait by returning null @@ -664,9 +665,6 @@ public class ConsoleProxyManagerImpl extends ManagerBase implements ConsoleProxy if (s_logger.isDebugEnabled()) { s_logger.debug("Unable to allocate console proxy storage, remove the console proxy record from DB, proxy id: " + proxyVmId); } - - SubscriptionMgr.getInstance().notifySubscribers(ConsoleProxyManager.ALERT_SUBJECT, this, - new ConsoleProxyAlertEventArgs(ConsoleProxyAlertEventArgs.PROXY_CREATE_FAILURE, dataCenterId, proxyVmId, null, "Unable to allocate storage")); } return null; } @@ -883,43 +881,57 @@ public class ConsoleProxyManagerImpl extends ManagerBase implements ConsoleProxy } ConsoleProxyVO proxy = null; - if (_allocProxyLock.lock(ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_SYNC)) { - try { - proxy = assignProxyFromStoppedPool(dataCenterId); - if (proxy == null) { - if (s_logger.isInfoEnabled()) { - s_logger.info("No stopped console proxy is available, need to allocate a new console proxy"); - } + String errorString = null; + try{ + if (_allocProxyLock.lock(ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_SYNC)) { + try { + proxy = assignProxyFromStoppedPool(dataCenterId); + if (proxy == null) { + if (s_logger.isInfoEnabled()) { + s_logger.info("No stopped console proxy is available, need to allocate a new console proxy"); + } - try { - proxy = startNew(dataCenterId); - } catch (ConcurrentOperationException e) { - s_logger.info("Concurrent Operation caught " + e); - } - } else { - if (s_logger.isInfoEnabled()) { - s_logger.info("Found a stopped console proxy, bring it up to running pool. proxy vm id : " + proxy.getId()); + try { + proxy = startNew(dataCenterId); + } catch (ConcurrentOperationException e) { + s_logger.info("Concurrent Operation caught " + e); + } + } else { + if (s_logger.isInfoEnabled()) { + s_logger.info("Found a stopped console proxy, bring it up to running pool. proxy vm id : " + proxy.getId()); + } } + } finally { + _allocProxyLock.unlock(); + } + } else { + if (s_logger.isInfoEnabled()) { + s_logger.info("Unable to acquire proxy allocation lock, skip for next time"); } - } finally { - _allocProxyLock.unlock(); } - } else { - if (s_logger.isInfoEnabled()) { - s_logger.info("Unable to acquire proxy allocation lock, skip for next time"); - } - } - - if (proxy != null) { - long proxyVmId = proxy.getId(); - proxy = startProxy(proxyVmId); if (proxy != null) { - if (s_logger.isInfoEnabled()) { - s_logger.info("Console proxy " + proxy.getHostName() + " is started"); + long proxyVmId = proxy.getId(); + proxy = startProxy(proxyVmId); + + if (proxy != null) { + if (s_logger.isInfoEnabled()) { + s_logger.info("Console proxy " + proxy.getHostName() + " is started"); + } + SubscriptionMgr.getInstance().notifySubscribers(ConsoleProxyManager.ALERT_SUBJECT, this, + new ConsoleProxyAlertEventArgs(ConsoleProxyAlertEventArgs.PROXY_UP, dataCenterId, proxy.getId(), proxy, null)); } } + }catch (Exception e){ + errorString = e.getMessage(); + }finally { + // TODO - For now put all the alerts as creation failure. Distinguish between creation vs start failure in future. + // Also add failure reason since startvm masks some of them. + if(proxy == null || proxy.getState() != State.Running) + SubscriptionMgr.getInstance().notifySubscribers(ConsoleProxyManager.ALERT_SUBJECT, this, + new ConsoleProxyAlertEventArgs(ConsoleProxyAlertEventArgs.PROXY_CREATE_FAILURE, dataCenterId, 0l, null, errorString)); } + } public boolean isZoneReady(Map zoneHostInfoMap, long dataCenterId) { diff --git a/services/secondary-storage/controller/src/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java b/services/secondary-storage/controller/src/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java index 25e5b85b599..9edd60059ed 100755 --- a/services/secondary-storage/controller/src/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java +++ b/services/secondary-storage/controller/src/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java @@ -506,9 +506,6 @@ public class SecondaryStorageManagerImpl extends ManagerBase implements Secondar s_logger.debug("Unable to allocate secondary storage vm storage, remove the secondary storage vm record from DB, secondary storage vm id: " + secStorageVmId); } - - SubscriptionMgr.getInstance().notifySubscribers(ALERT_SUBJECT, this, - new SecStorageVmAlertEventArgs(SecStorageVmAlertEventArgs.SSVM_CREATE_FAILURE, dataCenterId, secStorageVmId, null, "Unable to allocate storage")); } return null; } @@ -654,70 +651,83 @@ public class SecondaryStorageManagerImpl extends ManagerBase implements Secondar } return; } - - boolean secStorageVmFromStoppedPool = false; - SecondaryStorageVmVO secStorageVm = assignSecStorageVmFromStoppedPool(dataCenterId, role); - if (secStorageVm == null) { - if (s_logger.isInfoEnabled()) { - s_logger.info("No stopped secondary storage vm is available, need to allocate a new secondary storage vm"); - } - - if (_allocLock.lock(ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_SYNC)) { - try { - secStorageVm = startNew(dataCenterId, role); - for (UploadVO upload : _uploadDao.listAll()) { - _uploadDao.expunge(upload.getId()); - } - } finally { - _allocLock.unlock(); - } - } else { + SecondaryStorageVmVO secStorageVm = null; + String errorString = null; + try{ + boolean secStorageVmFromStoppedPool = false; + secStorageVm = assignSecStorageVmFromStoppedPool(dataCenterId, role); + if (secStorageVm == null) { if (s_logger.isInfoEnabled()) { - s_logger.info("Unable to acquire synchronization lock to allocate secStorageVm resource for standby capacity, wait for next scan"); + s_logger.info("No stopped secondary storage vm is available, need to allocate a new secondary storage vm"); } - return; - } - } else { - if (s_logger.isInfoEnabled()) { - s_logger.info("Found a stopped secondary storage vm, bring it up to running pool. secStorageVm vm id : " + secStorageVm.getId()); - } - secStorageVmFromStoppedPool = true; - } - if (secStorageVm != null) { - long secStorageVmId = secStorageVm.getId(); - GlobalLock secStorageVmLock = GlobalLock.getInternLock(getSecStorageVmLockName(secStorageVmId)); - try { - if (secStorageVmLock.lock(ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_SYNC)) { + if (_allocLock.lock(ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_SYNC)) { try { - secStorageVm = startSecStorageVm(secStorageVmId); + secStorageVm = startNew(dataCenterId, role); + for (UploadVO upload : _uploadDao.listAll()) { + _uploadDao.expunge(upload.getId()); + } } finally { - secStorageVmLock.unlock(); + _allocLock.unlock(); } } else { if (s_logger.isInfoEnabled()) { - s_logger.info("Unable to acquire synchronization lock to start secStorageVm for standby capacity, secStorageVm vm id : " + secStorageVm.getId()); + s_logger.info("Unable to acquire synchronization lock to allocate secStorageVm resource for standby capacity, wait for next scan"); } return; } - } finally { - secStorageVmLock.releaseRef(); - } - - if (secStorageVm == null) { - if (s_logger.isInfoEnabled()) { - s_logger.info("Unable to start secondary storage vm for standby capacity, secStorageVm vm Id : " + secStorageVmId + - ", will recycle it and start a new one"); - } - - if (secStorageVmFromStoppedPool) { - destroySecStorageVm(secStorageVmId); - } } else { if (s_logger.isInfoEnabled()) { - s_logger.info("Secondary storage vm " + secStorageVm.getHostName() + " is started"); + s_logger.info("Found a stopped secondary storage vm, bring it up to running pool. secStorageVm vm id : " + secStorageVm.getId()); + } + secStorageVmFromStoppedPool = true; + } + + if (secStorageVm != null) { + long secStorageVmId = secStorageVm.getId(); + GlobalLock secStorageVmLock = GlobalLock.getInternLock(getSecStorageVmLockName(secStorageVmId)); + try { + if (secStorageVmLock.lock(ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_SYNC)) { + try { + secStorageVm = startSecStorageVm(secStorageVmId); + } finally { + secStorageVmLock.unlock(); + } + } else { + if (s_logger.isInfoEnabled()) { + s_logger.info("Unable to acquire synchronization lock to start secStorageVm for standby capacity, secStorageVm vm id : " + secStorageVm.getId()); + } + return; + } + } finally { + secStorageVmLock.releaseRef(); + } + + if (secStorageVm == null) { + if (s_logger.isInfoEnabled()) { + s_logger.info("Unable to start secondary storage vm for standby capacity, secStorageVm vm Id : " + secStorageVmId + + ", will recycle it and start a new one"); + } + + if (secStorageVmFromStoppedPool) { + destroySecStorageVm(secStorageVmId); + } + } else { + SubscriptionMgr.getInstance().notifySubscribers(ALERT_SUBJECT, this, + new SecStorageVmAlertEventArgs(SecStorageVmAlertEventArgs.SSVM_UP, dataCenterId, secStorageVmId, secStorageVm, null)); + if (s_logger.isInfoEnabled()) { + s_logger.info("Secondary storage vm " + secStorageVm.getHostName() + " is started"); + } } } + }catch (Exception e){ + errorString = e.getMessage(); + }finally{ + // TODO - For now put all the alerts as creation failure. Distinguish between creation vs start failure in future. + // Also add failure reason since startvm masks some of them. + if(secStorageVm == null || secStorageVm.getState() != State.Running) + SubscriptionMgr.getInstance().notifySubscribers(ALERT_SUBJECT, this, + new SecStorageVmAlertEventArgs(SecStorageVmAlertEventArgs.SSVM_CREATE_FAILURE, dataCenterId, 0l, null, errorString)); } } From 665aa6b52289306c3085868d0ec2dfe1837bc13e Mon Sep 17 00:00:00 2001 From: Nitin Mehta Date: Fri, 14 Nov 2014 13:35:40 -0800 Subject: [PATCH 093/391] CLOUDSTACK-7916: Generate Alerts if System VMs cannot be started. (cherry picked from commit e87bb20d0c73e6f46f1352e271afec3961067542) Signed-off-by: Rohit Yadav --- server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java | 1 + .../cloudstack/secondarystorage/SecondaryStorageManagerImpl.java | 1 + 2 files changed, 2 insertions(+) diff --git a/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java b/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java index a527160e1c3..bb64fa3a12c 100755 --- a/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java +++ b/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java @@ -924,6 +924,7 @@ public class ConsoleProxyManagerImpl extends ManagerBase implements ConsoleProxy } }catch (Exception e){ errorString = e.getMessage(); + throw e; }finally { // TODO - For now put all the alerts as creation failure. Distinguish between creation vs start failure in future. // Also add failure reason since startvm masks some of them. diff --git a/services/secondary-storage/controller/src/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java b/services/secondary-storage/controller/src/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java index 9edd60059ed..cc7eb253bb0 100755 --- a/services/secondary-storage/controller/src/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java +++ b/services/secondary-storage/controller/src/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java @@ -722,6 +722,7 @@ public class SecondaryStorageManagerImpl extends ManagerBase implements Secondar } }catch (Exception e){ errorString = e.getMessage(); + throw e; }finally{ // TODO - For now put all the alerts as creation failure. Distinguish between creation vs start failure in future. // Also add failure reason since startvm masks some of them. From be0882b19e16b72cacc1a38b6bb72953b9053c5a Mon Sep 17 00:00:00 2001 From: Nitin Mehta Date: Thu, 13 Nov 2014 13:54:45 -0800 Subject: [PATCH 094/391] CLOUDSTACK-7909: Change the capacity_state in op_host_Capacity table only on explicit enabling/disabling of the host and remove the logic for changing it when cluster/pod/zone is enabled/disabled. Also add the logic in capacity checker thread so that previous changes finally get consistent with this new model and also its good to have it for sanity reasons. (cherry picked from commit ade305be20021d20a881693625864f46a335ea98) Signed-off-by: Rohit Yadav Conflicts: server/src/com/cloud/capacity/CapacityManagerImpl.java --- .../cloud/capacity/CapacityManagerImpl.java | 25 +++++++++++-------- .../ConfigurationManagerImpl.java | 6 ++--- .../cloud/resource/ResourceManagerImpl.java | 15 +++-------- 3 files changed, 21 insertions(+), 25 deletions(-) diff --git a/server/src/com/cloud/capacity/CapacityManagerImpl.java b/server/src/com/cloud/capacity/CapacityManagerImpl.java index e5b7d191e2c..5ea7e3e4b71 100755 --- a/server/src/com/cloud/capacity/CapacityManagerImpl.java +++ b/server/src/com/cloud/capacity/CapacityManagerImpl.java @@ -27,6 +27,8 @@ import javax.ejb.Local; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.resource.ResourceState; + import org.apache.log4j.Logger; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreDriver; @@ -49,7 +51,6 @@ import com.cloud.agent.api.Answer; import com.cloud.agent.api.Command; import com.cloud.agent.api.StartupCommand; import com.cloud.agent.api.StartupRoutingCommand; -import com.cloud.api.ApiDBUtils; import com.cloud.capacity.dao.CapacityDao; import com.cloud.configuration.Config; import com.cloud.configuration.ConfigurationManager; @@ -67,7 +68,6 @@ import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.hypervisor.dao.HypervisorCapabilitiesDao; import com.cloud.offering.ServiceOffering; -import com.cloud.org.Grouping.AllocationState; import com.cloud.resource.ResourceListener; import com.cloud.resource.ResourceManager; import com.cloud.resource.ServerResource; @@ -597,6 +597,7 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager, long usedMemory = 0; long reservedMemory = 0; long reservedCpu = 0; + final CapacityState capacityState = (host.getResourceState() == ResourceState.Enabled) ? CapacityState.Enabled : CapacityState.Disabled; List vms = _vmDao.listUpByHostId(host.getId()); if (s_logger.isDebugEnabled()) { @@ -680,6 +681,12 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager, cpuCap.setTotalCapacity(hostTotalCpu); } + // Set the capacity state as per the host allocation state. + if(capacityState != cpuCap.getCapacityState()){ + s_logger.debug("Calibrate cpu capacity state for host: " + host.getId() + " old capacity state:" + cpuCap.getTotalCapacity() + " new capacity state:" + hostTotalCpu); + cpuCap.setCapacityState(capacityState); + } + memCap.setCapacityState(capacityState); if (cpuCap.getUsedCapacity() == usedCpu && cpuCap.getReservedCapacity() == reservedCpu) { s_logger.debug("No need to calibrate cpu capacity, host:" + host.getId() + " usedCpu: " + cpuCap.getUsedCapacity() + " reservedCpu: " + @@ -702,6 +709,11 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager, memCap.setTotalCapacity(host.getTotalMemory()); } + // Set the capacity state as per the host allocation state. + if(capacityState != memCap.getCapacityState()){ + s_logger.debug("Calibrate memory capacity state for host: " + host.getId() + " old capacity state:" + cpuCap.getTotalCapacity() + " new capacity state:" + hostTotalCpu); + memCap.setCapacityState(capacityState); + } if (memCap.getUsedCapacity() == usedMemory && memCap.getReservedCapacity() == reservedMemory) { s_logger.debug("No need to calibrate memory capacity, host:" + host.getId() + " usedMem: " + memCap.getUsedCapacity() + " reservedMem: " + @@ -741,14 +753,7 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager, new CapacityVO(host.getId(), host.getDataCenterId(), host.getPodId(), host.getClusterId(), usedMemoryFinal, host.getTotalMemory(), Capacity.CAPACITY_TYPE_MEMORY); capacity.setReservedCapacity(reservedMemoryFinal); - CapacityState capacityState = CapacityState.Enabled; - if (host.getClusterId() != null) { - ClusterVO cluster = ApiDBUtils.findClusterById(host.getClusterId()); - if (cluster != null) { - capacityState = _configMgr.findClusterAllocationState(cluster) == AllocationState.Disabled ? CapacityState.Disabled : CapacityState.Enabled; - capacity.setCapacityState(capacityState); - } - } + capacity.setCapacityState(capacityState); _capacityDao.persist(capacity); capacity = diff --git a/server/src/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/com/cloud/configuration/ConfigurationManagerImpl.java index d9ce3fd3689..b18e1541723 100755 --- a/server/src/com/cloud/configuration/ConfigurationManagerImpl.java +++ b/server/src/com/cloud/configuration/ConfigurationManagerImpl.java @@ -1237,9 +1237,8 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati String ipRange = startIp + "-" + endIp; pod.setDescription(ipRange); Grouping.AllocationState allocationState = null; - if (allocationStateStrFinal != null && !allocationStateStrFinal.isEmpty()) { - allocationState = Grouping.AllocationState.valueOf(allocationStateStrFinal); - _capacityDao.updateCapacityState(null, pod.getId(), null, null, allocationStateStrFinal); + if (allocationStateStrFinal != null && !allocationStateStrFinal.isEmpty()) { + allocationState = Grouping.AllocationState.valueOf(allocationStateStrFinal); pod.setAllocationState(allocationState); } @@ -1774,7 +1773,6 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati throw new InvalidParameterValueException("Cannot enable this Zone since: " + ex.getMessage()); } } - _capacityDao.updateCapacityState(zone.getId(), null, null, null, allocationStateStr); zone.setAllocationState(allocationState); } diff --git a/server/src/com/cloud/resource/ResourceManagerImpl.java b/server/src/com/cloud/resource/ResourceManagerImpl.java index 5a1b8ce3f0e..57231964388 100755 --- a/server/src/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/com/cloud/resource/ResourceManagerImpl.java @@ -30,6 +30,7 @@ import javax.ejb.Local; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.capacity.CapacityState; import com.cloud.vm.VirtualMachine; import org.apache.cloudstack.api.ApiConstants; @@ -68,7 +69,6 @@ import com.cloud.agent.api.UpdateHostPasswordCommand; import com.cloud.agent.api.VgpuTypesInfo; import com.cloud.agent.api.to.GPUDeviceTO; import com.cloud.agent.transport.Request; -import com.cloud.api.ApiDBUtils; import com.cloud.capacity.Capacity; import com.cloud.capacity.CapacityManager; import com.cloud.capacity.CapacityVO; @@ -126,7 +126,6 @@ import com.cloud.network.dao.IPAddressDao; import com.cloud.network.dao.IPAddressVO; import com.cloud.org.Cluster; import com.cloud.org.Grouping; -import com.cloud.org.Grouping.AllocationState; import com.cloud.org.Managed; import com.cloud.serializer.GsonHelper; import com.cloud.service.dao.ServiceOfferingDetailsDao; @@ -1041,7 +1040,6 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, s_logger.error("Unable to resolve " + allocationState + " to a valid supported allocation State"); throw new InvalidParameterValueException("Unable to resolve " + allocationState + " to a supported state"); } else { - _capacityDao.updateCapacityState(null, null, cluster.getId(), null, allocationState); cluster.setAllocationState(newAllocationState); doUpdate = true; } @@ -1163,14 +1161,9 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, throw new NoTransitionException("No next resource state found for current state =" + currentState + " event =" + event); } - // TO DO - Make it more granular and have better conversion into - // capacity type - - if (host.getType() == Type.Routing && host.getClusterId() != null) { - AllocationState capacityState = _configMgr.findClusterAllocationState(ApiDBUtils.findClusterById(host.getClusterId())); - if (capacityState == AllocationState.Enabled && nextState != ResourceState.Enabled) { - capacityState = AllocationState.Disabled; - } + // TO DO - Make it more granular and have better conversion into capacity type + if(host.getType() == Type.Routing){ + CapacityState capacityState = (nextState == ResourceState.Enabled) ? CapacityState.Enabled : CapacityState.Disabled; _capacityDao.updateCapacityState(null, null, null, host.getId(), capacityState.toString()); } return _hostDao.updateResourceState(currentState, event, nextState, host); From b3e84cad2d37ef1d266bf49c0367dabe8589ace0 Mon Sep 17 00:00:00 2001 From: Sanjay Tripathi Date: Thu, 13 Nov 2014 18:56:29 +0530 Subject: [PATCH 095/391] CLOUDSTACK-7904: EN-UK: CentOS 6.5 GUI: Key translation fails for some EN- UK keyboard keys. (cherry picked from commit 7f2219bcbb4835123bdb24ba733651b9a1f2bdc6) Signed-off-by: Rohit Yadav --- systemvm/js/ajaxkeys.js | 43 ++++++++++----------------------------- systemvm/js/ajaxviewer.js | 4 +++- 2 files changed, 14 insertions(+), 33 deletions(-) diff --git a/systemvm/js/ajaxkeys.js b/systemvm/js/ajaxkeys.js index 26c29967c61..3053b7dbfa7 100644 --- a/systemvm/js/ajaxkeys.js +++ b/systemvm/js/ajaxkeys.js @@ -289,47 +289,26 @@ var keyboardTables = [ ] } }, {tindex: 2, keyboardType: KEYBOARD_TYPE_UK, mappingTable: - {X11: [ - //[223 = `¬¦] - {keycode: 223, entry : 0x60, browser: "IE"}, - ], + {X11: [], keyPress: [ - //[34 = "] - {keycode: 34, entry: - [{type : KEY_DOWN, code : 0x40, modifiers : 64, shift : true}] - }, + //[34 = "] + {keycode: 34, entry: 0x40, guestos: "windows"}, //[35 = #] - {keycode: 35, entry: - [{type : KEY_DOWN, code : 0x5c, modifiers : 0, shift : false}] - }, + {keycode: 35, entry: 0x5c, guestos: "windows"}, // [64 = @] - {keycode: 64, entry: - [{type : KEY_DOWN, code : 0x22, modifiers : 64, shift : true}] - }, + {keycode: 64, entry: 0x22, guestos: "windows"}, // [92 = \] - {keycode: 92, entry: - [{type : KEY_DOWN, code : 0xa6, modifiers : 0, shift : false}] - }, + {keycode: 92, entry: 0xa6, guestos: "windows"}, // [124 = |] - {keycode: 124, entry: - [{type : KEY_DOWN, code : 0xa6, modifiers : 64, shift : true}] - }, + {keycode: 124, entry: 0xa6, guestos: "windows"}, // [126 = ~] - {keycode: 126, entry: - [{type : KEY_DOWN, code : 0x7c, modifiers : 64, shift : true}] - }, + {keycode: 126, entry: 0x7c, guestos: "windows"}, // [163 = £] - {keycode: 163, entry: - [{type : KEY_DOWN, code : 0x23, modifiers : 64, shift : true}] - }, + {keycode: 163, entry: 0x23, guestos: "windows"}, // [172 = ¬] - {keycode: 172, entry: - [{type : KEY_DOWN, code : 0x7e, modifiers : 64, shift : true}] - }, + {keycode: 172, entry: 0x7e, guestos: "windows"}, // [166 = ¦] - {keycode: 166, entry: - [{type : KEY_DOWN, code : 0x60, modifiers : 896, shift : false}] - } + {keycode: 166, entry: [{type : KEY_DOWN, code : 0x60, modifiers : 896, shift : false}], guestos: "windows"} ] } }] diff --git a/systemvm/js/ajaxviewer.js b/systemvm/js/ajaxviewer.js index 123f1825c43..db5ed45b92c 100644 --- a/systemvm/js/ajaxviewer.js +++ b/systemvm/js/ajaxviewer.js @@ -761,7 +761,9 @@ AjaxViewer.prototype = { for (var j = 0; j < keyPressMaps.length; j++) { var code = keyPressMaps[j].keycode; var mappedEntry = keyPressMaps[j].entry; - this.keyboardMappers[keyboardType].jsKeyPressX11KeysymMap[code] = mappedEntry; + if(keyPressMaps[j].guestos == undefined || keyPressMaps[j].guestos == this.guestos) { + this.keyboardMappers[keyboardType].jsKeyPressX11KeysymMap[code] = mappedEntry; + } } } } From 1998217f6a4d00d27c88e559bd5889a003b39060 Mon Sep 17 00:00:00 2001 From: Gabor Apati-Nagy Date: Tue, 4 Nov 2014 18:32:32 +0000 Subject: [PATCH 096/391] CLOUDSTACK-7840: UI control tip for 'Add Primary Storage' -> 'Provider' seems wrong -Removed the invalid help text. (cherry picked from commit 1f21f399ab7711534febd77bb695e6c149293481) Signed-off-by: Rohit Yadav --- ui/scripts/system.js | 1 - 1 file changed, 1 deletion(-) diff --git a/ui/scripts/system.js b/ui/scripts/system.js index fdaaed4bdee..f55807afe32 100644 --- a/ui/scripts/system.js +++ b/ui/scripts/system.js @@ -17010,7 +17010,6 @@ }, provider: { label: 'label.provider', - docID: 'helpPrimaryStorageZone', validation: { required: true }, From 1288d533efe1c49c9dd0dbebee84b275b9f61f78 Mon Sep 17 00:00:00 2001 From: Rajesh Battala Date: Thu, 23 Oct 2014 14:14:14 +0530 Subject: [PATCH 097/391] CLOUDSTACK-7774 Description field is missing in Health policy API's (cherry picked from commit e5d3e496ab22be2bcbb7f92e6c80bfad8e503153) Signed-off-by: Rohit Yadav --- .../cloudstack/api/response/LBHealthCheckPolicyResponse.java | 1 + 1 file changed, 1 insertion(+) diff --git a/api/src/org/apache/cloudstack/api/response/LBHealthCheckPolicyResponse.java b/api/src/org/apache/cloudstack/api/response/LBHealthCheckPolicyResponse.java index 8ce7b3d219b..df3a7a0fb7b 100644 --- a/api/src/org/apache/cloudstack/api/response/LBHealthCheckPolicyResponse.java +++ b/api/src/org/apache/cloudstack/api/response/LBHealthCheckPolicyResponse.java @@ -101,6 +101,7 @@ public class LBHealthCheckPolicyResponse extends BaseResponse { this.healthcheckthresshold = healthcheckpolicy.getHealthcheckThresshold(); this.unhealthcheckthresshold = healthcheckpolicy.getUnhealthThresshold(); this.forDisplay = healthcheckpolicy.isDisplay(); + this.description = healthcheckpolicy.getDescription(); setObjectName("healthcheckpolicy"); } From a6f9411b1eb46739c9b47fc16ad7796cc0e3d221 Mon Sep 17 00:00:00 2001 From: Nitin Mehta Date: Wed, 22 Oct 2014 15:03:25 -0700 Subject: [PATCH 098/391] CLOUDSTACK-7771: Throw exception while restoring vm if the template is already deleted. Also fix the usage event being generated from the right template id and fix the log message which sometime showed incorrect template id. CLOUDSTACK-4201: listServiceOfferings API needs to be able to take virtualmachineid of SystemVM and return service offerings available for the vm to change service offering (cherry picked from commit ee0f0a1cffb96a8371d9a8c4210b428660f79d4f) Signed-off-by: Rohit Yadav --- server/src/com/cloud/api/query/QueryManagerImpl.java | 7 ++++++- server/src/com/cloud/vm/UserVmManagerImpl.java | 12 ++++++++++-- server/test/com/cloud/vm/UserVmManagerTest.java | 8 ++++++++ 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/server/src/com/cloud/api/query/QueryManagerImpl.java b/server/src/com/cloud/api/query/QueryManagerImpl.java index a5e7d5c5612..62714eafab2 100644 --- a/server/src/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/com/cloud/api/query/QueryManagerImpl.java @@ -27,6 +27,8 @@ import javax.ejb.Local; import javax.inject.Inject; import com.cloud.utils.StringUtils; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.dao.VMInstanceDao; import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.affinity.AffinityGroupDomainMapVO; import org.apache.cloudstack.affinity.AffinityGroupResponse; @@ -247,6 +249,9 @@ public class QueryManagerImpl extends ManagerBase implements QueryService { @Inject private UserVmDao _userVmDao; + @Inject + private VMInstanceDao _vmInstanceDao; + @Inject private SecurityGroupJoinDao _securityGroupJoinDao; @@ -2576,7 +2581,7 @@ public class QueryManagerImpl extends ManagerBase implements QueryService { } if (vmId != null) { - UserVmVO vmInstance = _userVmDao.findById(vmId); + VMInstanceVO vmInstance = _vmInstanceDao.findById(vmId); if ((vmInstance == null) || (vmInstance.getRemoved() != null)) { InvalidParameterValueException ex = new InvalidParameterValueException("unable to find a virtual machine with specified id"); ex.addProxyObject(vmId.toString(), "vmId"); diff --git a/server/src/com/cloud/vm/UserVmManagerImpl.java b/server/src/com/cloud/vm/UserVmManagerImpl.java index 3f539b25567..22f27691c07 100755 --- a/server/src/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/com/cloud/vm/UserVmManagerImpl.java @@ -35,6 +35,8 @@ import javax.ejb.Local; import javax.inject.Inject; import javax.naming.ConfigurationException; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import org.apache.commons.codec.binary.Base64; import org.apache.log4j.Logger; import org.apache.cloudstack.acl.ControlledEntity.ACLType; @@ -303,6 +305,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir @Inject protected VMTemplateZoneDao _templateZoneDao = null; @Inject + protected TemplateDataStoreDao _templateStoreDao; + @Inject protected DomainDao _domainDao = null; @Inject protected UserVmCloneSettingDao _vmCloneSettingDao = null; @@ -4762,6 +4766,10 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir throw ex; } } + TemplateDataStoreVO tmplStore = _templateStoreDao.findByTemplateZoneReady(template.getId(), vm.getDataCenterId()); + if (tmplStore == null) { + throw new InvalidParameterValueException("Cannot restore the vm as the template " + template.getUuid() + " isn't available in the zone"); + } if (needRestart) { try { @@ -4794,7 +4802,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir } // Create Usage event for the newly created volume - UsageEventVO usageEvent = new UsageEventVO(EventTypes.EVENT_VOLUME_CREATE, newVol.getAccountId(), newVol.getDataCenterId(), newVol.getId(), newVol.getName(), newVol.getDiskOfferingId(), templateId, newVol.getSize()); + UsageEventVO usageEvent = new UsageEventVO(EventTypes.EVENT_VOLUME_CREATE, newVol.getAccountId(), newVol.getDataCenterId(), newVol.getId(), newVol.getName(), newVol.getDiskOfferingId(), template.getId(), newVol.getSize()); _usageEventDao.persist(usageEvent); handleManagedStorage(vm, root); @@ -4867,7 +4875,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir } } - s_logger.debug("Restore VM " + vmId + " with template " + newTemplateId + " done successfully"); + s_logger.debug("Restore VM " + vmId + " done successfully"); return vm; } diff --git a/server/test/com/cloud/vm/UserVmManagerTest.java b/server/test/com/cloud/vm/UserVmManagerTest.java index aed468d3b8b..d86e55ee49b 100755 --- a/server/test/com/cloud/vm/UserVmManagerTest.java +++ b/server/test/com/cloud/vm/UserVmManagerTest.java @@ -55,6 +55,7 @@ import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationSer import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import com.cloud.capacity.CapacityManager; import com.cloud.configuration.ConfigurationManager; @@ -89,6 +90,7 @@ import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.VMInstanceDao; import com.cloud.vm.snapshot.VMSnapshotVO; import com.cloud.vm.snapshot.dao.VMSnapshotDao; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; public class UserVmManagerTest { @@ -121,6 +123,8 @@ public class UserVmManagerTest { @Mock VMTemplateDao _templateDao; @Mock + TemplateDataStoreDao _templateStoreDao; + @Mock VolumeDao _volsDao; @Mock RestoreVMCmd _restoreVMCmd; @@ -135,6 +139,8 @@ public class UserVmManagerTest { @Mock VMTemplateVO _templateMock; @Mock + TemplateDataStoreVO _templateDataStoreMock; + @Mock VolumeVO _volumeMock; @Mock List _rootVols; @@ -162,6 +168,7 @@ public class UserVmManagerTest { _userVmMgr._vmDao = _vmDao; _userVmMgr._vmInstanceDao = _vmInstanceDao; _userVmMgr._templateDao = _templateDao; + _userVmMgr._templateStoreDao = _templateStoreDao; _userVmMgr._volsDao = _volsDao; _userVmMgr._usageEventDao = _usageEventDao; _userVmMgr._itMgr = _itMgr; @@ -190,6 +197,7 @@ public class UserVmManagerTest { List mockList = mock(List.class); when(_vmSnapshotDao.findByVm(anyLong())).thenReturn(mockList); when(mockList.size()).thenReturn(0); + when(_templateStoreDao.findByTemplateZoneReady(anyLong(),anyLong())).thenReturn(_templateDataStoreMock); } From 31a6517402d2007a4857d3ae9be2da8a697892b1 Mon Sep 17 00:00:00 2001 From: Nitin Mehta Date: Tue, 21 Oct 2014 09:57:02 -0700 Subject: [PATCH 099/391] CLOUDSTACK-7754: Templates source_template_id is null when it is created from Snapshot with its corresponding volume removed. Fix it by searching for volumes including removed. Also bring the logic of setting source template id to create() method than execute which was wrongly put in. (cherry picked from commit e1a993d37b1bf9be24edd46f6d3566826692f853) Signed-off-by: Rohit Yadav --- .../cloud/template/TemplateManagerImpl.java | 40 +++---------------- 1 file changed, 6 insertions(+), 34 deletions(-) diff --git a/server/src/com/cloud/template/TemplateManagerImpl.java b/server/src/com/cloud/template/TemplateManagerImpl.java index 5641f577f3a..a498dc747b8 100755 --- a/server/src/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/com/cloud/template/TemplateManagerImpl.java @@ -1412,29 +1412,6 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, } privateTemplate = _tmpltDao.findById(templateId); - if (snapshotId != null) { - //getting the parent volume - long parentVolumeId = _snapshotDao.findById(snapshotId).getVolumeId(); - //Volume can be removed - VolumeVO parentVolume = _volumeDao.findByIdIncludingRemoved(parentVolumeId); - - if (parentVolume != null && parentVolume.getIsoId() != null && parentVolume.getIsoId() != 0) { - privateTemplate.setSourceTemplateId(parentVolume.getIsoId()); - _tmpltDao.update(privateTemplate.getId(), privateTemplate); - } else if (parentVolume != null && parentVolume.getTemplateId() != null) { - privateTemplate.setSourceTemplateId(parentVolume.getTemplateId()); - _tmpltDao.update(privateTemplate.getId(), privateTemplate); - } - } else if (volumeId != null) { - VolumeVO parentVolume = _volumeDao.findById(volumeId); - if (parentVolume.getIsoId() != null && parentVolume.getIsoId() != 0) { - privateTemplate.setSourceTemplateId(parentVolume.getIsoId()); - _tmpltDao.update(privateTemplate.getId(), privateTemplate); - } else if (parentVolume.getTemplateId() != null) { - privateTemplate.setSourceTemplateId(parentVolume.getTemplateId()); - _tmpltDao.update(privateTemplate.getId(), privateTemplate); - } - } TemplateDataStoreVO srcTmpltStore = _tmplStoreDao.findByStoreTemplate(store.getId(), templateId); UsageEventVO usageEvent = new UsageEventVO(EventTypes.EVENT_TEMPLATE_CREATE, privateTemplate.getAccountId(), zoneId, privateTemplate.getId(), privateTemplate.getName(), null, @@ -1570,8 +1547,8 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, if (snapshot == null) { throw new InvalidParameterValueException("Failed to create private template record, unable to find snapshot " + snapshotId); } - - volume = _volumeDao.findById(snapshot.getVolumeId()); + // Volume could be removed so find including removed to record source template id. + volume = _volumeDao.findByIdIncludingRemoved(snapshot.getVolumeId()); // check permissions _accountMgr.checkAccess(caller, null, true, snapshot); @@ -1612,15 +1589,10 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, if (volume != null) { VMTemplateVO template = ApiDBUtils.findTemplateById(volume.getTemplateId()); isExtractable = template != null && template.isExtractable() && template.getTemplateType() != Storage.TemplateType.SYSTEM; - if (template != null) { - sourceTemplateId = template.getId(); - } else if (volume.getVolumeType() == Volume.Type.ROOT) { // vm - // created - // out - // of blank - // template - UserVm userVm = ApiDBUtils.findUserVmById(volume.getInstanceId()); - sourceTemplateId = userVm.getIsoId(); + if (volume.getIsoId() != null && volume.getIsoId() != 0) { + sourceTemplateId = volume.getIsoId(); + } else if (volume.getTemplateId() != null) { + sourceTemplateId = volume.getTemplateId(); } } String templateTag = cmd.getTemplateTag(); From 45108fdbadaf0e0b8d6a98a730a27ab00bcca3b7 Mon Sep 17 00:00:00 2001 From: Saksham Srivastava Date: Wed, 22 Oct 2014 13:53:41 +0530 Subject: [PATCH 100/391] CLOUDSTACK-7582: Update Storage Pool API does not update tags correctly (cherry picked from commit fc4dceaa991ecacf0d248725decd5370622ea0ed) Signed-off-by: Rohit Yadav --- .../storage/datastore/db/PrimaryDataStoreDaoImpl.java | 3 +++ server/src/com/cloud/storage/StorageManagerImpl.java | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/engine/schema/src/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java b/engine/schema/src/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java index 0c372a5b99a..ae2287e9181 100644 --- a/engine/schema/src/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java +++ b/engine/schema/src/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java @@ -435,6 +435,9 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase detailsVO.add(new StoragePoolDetailVO(poolId, key, details.get(key), true)); } _detailsDao.saveDetails(detailsVO); + if(details.size() == 0) { + _detailsDao.removeDetails(poolId); + } } } diff --git a/server/src/com/cloud/storage/StorageManagerImpl.java b/server/src/com/cloud/storage/StorageManagerImpl.java index 7de4ea0c844..5ea44a2330f 100755 --- a/server/src/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/com/cloud/storage/StorageManagerImpl.java @@ -803,7 +803,7 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C } } - if (updatedDetails.size() > 0) { + if (updatedDetails.size() >= 0) { _storagePoolDao.updateDetails(id, updatedDetails); } From bf8db0c743c94c4540dab2f069d253f1a859eb10 Mon Sep 17 00:00:00 2001 From: Bharat Kumar Date: Wed, 17 Sep 2014 14:30:33 +0530 Subject: [PATCH 101/391] CLOUDSTACK-7571 changing value of cpu/mem.overprovisioning.factor for xen cluster is not affecting total memory at zone level (cherry picked from commit 476733cb92634c8494fe64762d7fbc178292a754) Signed-off-by: Rohit Yadav --- .../cloud/capacity/dao/CapacityDaoImpl.java | 32 ++++++++++++++----- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/engine/schema/src/com/cloud/capacity/dao/CapacityDaoImpl.java b/engine/schema/src/com/cloud/capacity/dao/CapacityDaoImpl.java index cea18bd2679..85740a22775 100755 --- a/engine/schema/src/com/cloud/capacity/dao/CapacityDaoImpl.java +++ b/engine/schema/src/com/cloud/capacity/dao/CapacityDaoImpl.java @@ -113,7 +113,7 @@ public class CapacityDaoImpl extends GenericDaoBase implements + "else sum(total_capacity) end)," + "((sum(capacity.used_capacity) + sum(capacity.reserved_capacity)) / ( case capacity_type when 1 then (sum(total_capacity) * (select value from `cloud`.`cluster_details` where cluster_details.name= 'cpuOvercommitRatio' AND cluster_details.cluster_id=capacity.cluster_id))" + "when '0' then (sum(total_capacity) * (select value from `cloud`.`cluster_details` where cluster_details.name='memoryOvercommitRatio' AND cluster_details.cluster_id=capacity.cluster_id))else sum(total_capacity) end)) percent," - + "capacity.capacity_type, capacity.data_center_id FROM `cloud`.`op_host_capacity` capacity WHERE total_capacity > 0 AND data_center_id is not null AND capacity_state='Enabled'"; + + "capacity.capacity_type, capacity.data_center_id, pod_id, cluster_id FROM `cloud`.`op_host_capacity` capacity WHERE total_capacity > 0 AND data_center_id is not null AND capacity_state='Enabled'"; private static final String LIST_CAPACITY_GROUP_BY_ZONE_TYPE_PART2 = " GROUP BY data_center_id, capacity_type order by percent desc limit "; private static final String LIST_CAPACITY_GROUP_BY_POD_TYPE_PART1 = @@ -122,7 +122,7 @@ public class CapacityDaoImpl extends GenericDaoBase implements + "when '0' then (sum(total_capacity) * (select value from `cloud`.`cluster_details` where cluster_details.name= 'memoryOvercommitRatio' AND cluster_details.cluster_id=capacity.cluster_id))else sum(total_capacity) end)," + "((sum(capacity.used_capacity) + sum(capacity.reserved_capacity)) / ( case capacity_type when 1 then (sum(total_capacity) * (select value from `cloud`.`cluster_details` where cluster_details.name= 'cpuOvercommitRatio' AND cluster_details.cluster_id=capacity.cluster_id)) " + "when '0' then (sum(total_capacity) * (select value from `cloud`.`cluster_details` where cluster_details.name= 'memoryOvercommitRatio' AND cluster_details.cluster_id=capacity.cluster_id))else sum(total_capacity) end)) percent," - + "capacity.capacity_type, capacity.data_center_id, pod_id FROM `cloud`.`op_host_capacity` capacity WHERE total_capacity > 0 AND data_center_id is not null AND capacity_state='Enabled' "; + + "capacity.capacity_type, capacity.data_center_id, pod_id, cluster_id FROM `cloud`.`op_host_capacity` capacity WHERE total_capacity > 0 AND data_center_id is not null AND capacity_state='Enabled' "; private static final String LIST_CAPACITY_GROUP_BY_POD_TYPE_PART2 = " GROUP BY pod_id, capacity_type order by percent desc limit "; @@ -134,7 +134,7 @@ public class CapacityDaoImpl extends GenericDaoBase implements + "when '0' then (sum(total_capacity) * (select value from `cloud`.`cluster_details` where cluster_details.name= 'memoryOvercommitRatio' AND cluster_details.cluster_id=capacity.cluster_id))else sum(total_capacity) end)) percent," + "capacity.capacity_type, capacity.data_center_id, pod_id, cluster_id FROM `cloud`.`op_host_capacity` capacity WHERE total_capacity > 0 AND data_center_id is not null AND capacity_state='Enabled' "; - private static final String LIST_CAPACITY_GROUP_BY_CLUSTER_TYPE_PART2 = " GROUP BY cluster_id, capacity_type order by percent desc limit "; + private static final String LIST_CAPACITY_GROUP_BY_CLUSTER_TYPE_PART2 = " GROUP BY cluster_id, capacity_type, pod_id order by percent desc limit "; private static final String UPDATE_CAPACITY_STATE = "UPDATE `cloud`.`op_host_capacity` SET capacity_state = ? WHERE "; private static final String LIST_CAPACITY_GROUP_BY_CAPACITY_PART1= "SELECT sum(capacity.used_capacity), sum(capacity.reserved_capacity)," + @@ -317,7 +317,7 @@ public class CapacityDaoImpl extends GenericDaoBase implements StringBuilder finalQuery = new StringBuilder(); TransactionLegacy txn = TransactionLegacy.currentTxn(); PreparedStatement pstmt = null; - List result = new ArrayList(); + List results = new ArrayList(); List resourceIdList = new ArrayList(); @@ -354,11 +354,11 @@ public class CapacityDaoImpl extends GenericDaoBase implements switch (level) { case 1: // List all the capacities grouped by zone, capacity Type - finalQuery.append(LIST_CAPACITY_GROUP_BY_ZONE_TYPE_PART2); + finalQuery.append(LIST_CAPACITY_GROUP_BY_CLUSTER_TYPE_PART2); break; case 2: // List all the capacities grouped by pod, capacity Type - finalQuery.append(LIST_CAPACITY_GROUP_BY_POD_TYPE_PART2); + finalQuery.append(LIST_CAPACITY_GROUP_BY_CLUSTER_TYPE_PART2); break; case 3: // List all the capacities grouped by cluster, capacity Type @@ -387,9 +387,25 @@ public class CapacityDaoImpl extends GenericDaoBase implements SummedCapacity summedCapacity = new SummedCapacity(rs.getLong(1), rs.getLong(2), rs.getLong(3), rs.getFloat(4), (short)rs.getLong(5), rs.getLong(6), capacityPodId, capacityClusterId); - result.add(summedCapacity); + results.add(summedCapacity); } - return result; + + HashMap capacityMap = new HashMap(); + for (SummedCapacity result: results) { + if (capacityMap.containsKey(result.getCapacityType().intValue())) { + SummedCapacity tempCapacity = capacityMap.get(result.getCapacityType().intValue()); + tempCapacity.setUsedCapacity(tempCapacity.getUsedCapacity()+result.getUsedCapacity()); + tempCapacity.setReservedCapacity(tempCapacity.getReservedCapacity()+result.getReservedCapacity()); + tempCapacity.setSumTotal(tempCapacity.getTotalCapacity()+result.getTotalCapacity()); + }else { + capacityMap.put(result.getCapacityType().intValue(),result); + } + } + List summedCapacityList = new ArrayList(); + for (Integer capacity_type : capacityMap.keySet()) { + summedCapacityList.add(capacityMap.get(capacity_type)); + } + return summedCapacityList; } catch (SQLException e) { throw new CloudRuntimeException("DB Exception on: " + finalQuery, e); } catch (Throwable e) { From 9cf05dc842f3fb4649746ad536eca1855184e15b Mon Sep 17 00:00:00 2001 From: Harikrishna Patnala Date: Thu, 11 Sep 2014 11:07:43 +0530 Subject: [PATCH 102/391] CLOUDSTACK-7534: ResetVM for VM with attached datadisk fails when enable.ha.storage.migration is false Separate global config to enable/disable Storage Migration during normal deployment Introduced a configuration parameter named enable.storage.migration (cherry picked from commit c55bc0b2d11be4820a24af426e23da3db54a0cb1) Signed-off-by: Rohit Yadav --- api/src/com/cloud/vm/VirtualMachineProfile.java | 1 + .../engine/orchestration/VolumeOrchestrator.java | 16 +++++++++++++--- .../cloud/ha/HighAvailabilityManagerImpl.java | 4 ++++ 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/api/src/com/cloud/vm/VirtualMachineProfile.java b/api/src/com/cloud/vm/VirtualMachineProfile.java index 29f31646321..d0fea49ae8d 100644 --- a/api/src/com/cloud/vm/VirtualMachineProfile.java +++ b/api/src/com/cloud/vm/VirtualMachineProfile.java @@ -41,6 +41,7 @@ public interface VirtualMachineProfile { public static final Param ReProgramGuestNetworks = new Param("RestartNetwork"); public static final Param PxeSeverType = new Param("PxeSeverType"); public static final Param HaTag = new Param("HaTag"); + public static final Param HaOperation = new Param("HaOperation"); private String name; diff --git a/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java index fd4883b1f7f..1a3c0b6dd86 100644 --- a/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java +++ b/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java @@ -1101,7 +1101,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati } } - private List getTasks(List vols, Map destVols) throws StorageUnavailableException { + private List getTasks(List vols, Map destVols, VirtualMachineProfile vm) throws StorageUnavailableException { boolean recreate = RecreatableSystemVmEnabled.value(); List tasks = new ArrayList(); for (VolumeVO vol : vols) { @@ -1141,7 +1141,14 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati throw new CloudRuntimeException("Local volume " + vol + " cannot be recreated on storagepool " + assignedPool + " assigned by deploymentPlanner"); } else { //Check if storage migration is enabled in config - if (StorageHAMigrationEnabled.value()) { + Boolean isHAOperation = (Boolean)vm.getParameter(VirtualMachineProfile.Param.HaOperation); + Boolean storageMigrationEnabled = true; + if (isHAOperation != null && isHAOperation) { + storageMigrationEnabled = StorageHAMigrationEnabled.value(); + } else { + storageMigrationEnabled = StorageMigrationEnabled.value(); + } + if(storageMigrationEnabled){ if (s_logger.isDebugEnabled()) { s_logger.debug("Shared volume " + vol + " will be migrated on storage pool " + assignedPool + " assigned by deploymentPlanner"); } @@ -1299,7 +1306,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati s_logger.debug("Checking if we need to prepare " + vols.size() + " volumes for " + vm); } - List tasks = getTasks(vols, dest.getStorageForDisks()); + List tasks = getTasks(vols, dest.getStorageForDisks(), vm); Volume vol = null; StoragePool pool = null; for (VolumeTask task : tasks) { @@ -1350,6 +1357,9 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati public static final ConfigKey StorageHAMigrationEnabled = new ConfigKey(Boolean.class, "enable.ha.storage.migration", "Storage", "true", "Enable/disable storage migration across primary storage during HA", true); + public static final ConfigKey StorageMigrationEnabled = new ConfigKey(Boolean.class, "enable.storage.migration", "Storage", "true", + "Enable/disable storage migration across primary storage", true); + @Override public ConfigKey[] getConfigKeys() { return new ConfigKey[] {RecreatableSystemVmEnabled, MaxVolumeSize, StorageHAMigrationEnabled, CustomDiskOfferingMaxSize, CustomDiskOfferingMinSize}; diff --git a/server/src/com/cloud/ha/HighAvailabilityManagerImpl.java b/server/src/com/cloud/ha/HighAvailabilityManagerImpl.java index a305543ca91..a278750d896 100755 --- a/server/src/com/cloud/ha/HighAvailabilityManagerImpl.java +++ b/server/src/com/cloud/ha/HighAvailabilityManagerImpl.java @@ -578,6 +578,10 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements HighAvai if (_haTag != null) { params.put(VirtualMachineProfile.Param.HaTag, _haTag); } + WorkType wt = work.getWorkType(); + if (wt.equals(WorkType.HA)) { + params.put(VirtualMachineProfile.Param.HaOperation, true); + } try{ // First try starting the vm with its original planner, if it doesn't succeed send HAPlanner as its an emergency. From 45ebdf34aee51217bf32e58039da16870dd1e5b3 Mon Sep 17 00:00:00 2001 From: Alex Brett Date: Mon, 13 Oct 2014 12:18:41 +0100 Subject: [PATCH 103/391] CLOUDSTACK-7469 Complete simulator build support The initial commit (f96c65416a2802bcf2a1f8d5a5070ffe6a29111f) missed part of the change to package.sh, so we were not actually passing through the simulator build option to the rpmbuild call. This patch completes the support. (cherry picked from commit e717450e0edd2406c4c3fc7341b3669c4390d507) Signed-off-by: Rohit Yadav --- packaging/centos63/package.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/centos63/package.sh b/packaging/centos63/package.sh index bbc3736d77e..00c19478ea0 100755 --- a/packaging/centos63/package.sh +++ b/packaging/centos63/package.sh @@ -79,7 +79,7 @@ function packaging() { cp -rf default $RPMDIR/SPECS cp -rf rhel7 $RPMDIR/SPECS - (cd $RPMDIR; rpmbuild --define "_topdir $RPMDIR" "${DEFVER}" "${DEFREL}" ${DEFPRE+"${DEFPRE}"} ${DEFOSSNOSS+"$DEFOSSNOSS"} "${DOS}" -bb SPECS/cloud.spec) + (cd $RPMDIR; rpmbuild --define "_topdir $RPMDIR" "${DEFVER}" "${DEFREL}" ${DEFPRE+"${DEFPRE}"} ${DEFOSSNOSS+"$DEFOSSNOSS"} ${DEFSIM+"$DEFSIM"} "${DOS}" -bb SPECS/cloud.spec) if [ $? -ne 0 ]; then echo "RPM Build Failed " From ed69d588ec56735451f2427c48bcfc499f863b9c Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Wed, 21 Jan 2015 16:00:29 +0530 Subject: [PATCH 104/391] CLOUDSTACK-5946: Fix keystore filename to avoid warning logs Signed-off-by: Rohit Yadav --- utils/src/com/cloud/utils/nio/Link.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/src/com/cloud/utils/nio/Link.java b/utils/src/com/cloud/utils/nio/Link.java index a15b8a4e1b6..df2965afd6b 100755 --- a/utils/src/com/cloud/utils/nio/Link.java +++ b/utils/src/com/cloud/utils/nio/Link.java @@ -65,7 +65,7 @@ public class Link { private boolean _gotFollowingPacket; private SSLEngine _sslEngine; - public static String keystoreFile = "/cloud.keystore"; + public static String keystoreFile = "/cloudmanagementserver.keystore"; public Link(InetSocketAddress addr, NioConnection connection) { _addr = addr; From debfcdef788ce0d51be06db0ef10f6815f9b563b Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Wed, 21 Jan 2015 18:01:34 +0530 Subject: [PATCH 105/391] CLOUDSTACK-8160: use preferable protocols Signed-off-by: Rohit Yadav --- client/tomcatconf/server-nonssl.xml.in | 2 +- client/tomcatconf/server-ssl.xml.in | 2 +- client/tomcatconf/server7-nonssl.xml.in | 2 +- client/tomcatconf/server7-ssl.xml.in | 2 +- .../manager/ClusteredAgentManagerImpl.java | 2 + .../mom/rabbitmq/RabbitMQEventBus.java | 7 ++- .../resource/XenServerConnectionPool.java | 4 +- .../opendaylight/api/NeutronRestApi.java | 19 +++++-- .../network/utils/HttpClientWrapper.java | 4 +- .../datastore/util/ElastistorUtil.java | 3 +- .../datastore/util/NexentaNmsClient.java | 4 +- .../storage/datastore/util/SolidFireUtil.java | 4 +- pom.xml | 7 +-- .../main/java/streamer/SocketWrapperImpl.java | 2 +- .../ConsoleProxySecureServerFactoryImpl.java | 6 ++- .../com/cloud/consoleproxy/util/RawHTTP.java | 25 +++++---- .../etc/apache2/sites-available/default-ssl | 1 + .../config/etc/apache2/vhostexample.conf | 1 + systemvm/scripts/config_ssl.sh | 2 + test/pom.xml | 2 +- utils/src/com/cloud/utils/nio/Link.java | 5 +- utils/src/com/cloud/utils/nio/NioClient.java | 3 ++ .../com/cloud/utils/nio/NioConnection.java | 3 ++ .../utils/rest/RESTServiceConnector.java | 20 ++++++-- .../cloudstack/utils/security/SSLUtils.java | 51 +++++++++++++++++++ .../ssl/EasySSLProtocolSocketFactory.java | 24 ++++++--- .../hypervisor/vmware/util/VmwareClient.java | 4 +- .../hypervisor/vmware/util/VmwareContext.java | 3 +- 28 files changed, 162 insertions(+), 52 deletions(-) create mode 100644 utils/src/org/apache/cloudstack/utils/security/SSLUtils.java diff --git a/client/tomcatconf/server-nonssl.xml.in b/client/tomcatconf/server-nonssl.xml.in index 847197cbae0..e0debe4a31e 100755 --- a/client/tomcatconf/server-nonssl.xml.in +++ b/client/tomcatconf/server-nonssl.xml.in @@ -82,7 +82,7 @@ diff --git a/services/console-proxy-rdp/rdpconsole/src/main/java/rdpclient/ntlmssp/ClientNtlmsspPubKeyAuth.java b/services/console-proxy-rdp/rdpconsole/src/main/java/rdpclient/ntlmssp/ClientNtlmsspPubKeyAuth.java index ffd16304df0..7aae145237b 100755 --- a/services/console-proxy-rdp/rdpconsole/src/main/java/rdpclient/ntlmssp/ClientNtlmsspPubKeyAuth.java +++ b/services/console-proxy-rdp/rdpconsole/src/main/java/rdpclient/ntlmssp/ClientNtlmsspPubKeyAuth.java @@ -16,6 +16,8 @@ // under the License. package rdpclient.ntlmssp; +import com.cloud.utils.ConstantTimeComparator; + import java.nio.charset.Charset; import rdpclient.ntlmssp.asn1.NegoItem; @@ -604,7 +606,7 @@ public class ClientNtlmsspPubKeyAuth extends OneTimeSwitch implements NtlmConsta private void dumpNegoToken(ByteBuffer buf) { String signature = buf.readVariableString(RdpConstants.CHARSET_8); - if (!signature.equals(NTLMSSP)) + if (!ConstantTimeComparator.compareStrings(signature, NTLMSSP)) throw new RuntimeException("Unexpected NTLM message singature: \"" + signature + "\". Expected signature: \"" + NTLMSSP + "\". Data: " + buf + "."); // MessageType (CHALLENGE) diff --git a/services/console-proxy-rdp/rdpconsole/src/main/java/rdpclient/ntlmssp/ServerNtlmsspChallenge.java b/services/console-proxy-rdp/rdpconsole/src/main/java/rdpclient/ntlmssp/ServerNtlmsspChallenge.java index eaac62b92d9..d8cfb4c9594 100755 --- a/services/console-proxy-rdp/rdpconsole/src/main/java/rdpclient/ntlmssp/ServerNtlmsspChallenge.java +++ b/services/console-proxy-rdp/rdpconsole/src/main/java/rdpclient/ntlmssp/ServerNtlmsspChallenge.java @@ -16,6 +16,8 @@ // under the License. package rdpclient.ntlmssp; +import com.cloud.utils.ConstantTimeComparator; + import java.util.Arrays; import rdpclient.ntlmssp.asn1.NegoItem; @@ -70,7 +72,7 @@ public class ServerNtlmsspChallenge extends OneTimeSwitch implements NtlmConstan // Signature: "NTLMSSP\0" String signature = buf.readVariableString(RdpConstants.CHARSET_8); - if (!signature.equals(NTLMSSP)) + if (!ConstantTimeComparator.compareStrings(signature, NTLMSSP)) throw new RuntimeException("Unexpected NTLM message singature: \"" + signature + "\". Expected signature: \"" + NTLMSSP + "\". Data: " + buf + "."); // MessageType (CHALLENGE) diff --git a/services/console-proxy-rdp/rdpconsole/src/main/java/streamer/SocketWrapperImpl.java b/services/console-proxy-rdp/rdpconsole/src/main/java/streamer/SocketWrapperImpl.java index abb5b84c1d5..6d996286a36 100755 --- a/services/console-proxy-rdp/rdpconsole/src/main/java/streamer/SocketWrapperImpl.java +++ b/services/console-proxy-rdp/rdpconsole/src/main/java/streamer/SocketWrapperImpl.java @@ -32,6 +32,8 @@ import javax.net.ssl.SSLSocket; import javax.net.ssl.SSLSocketFactory; import javax.net.ssl.TrustManager; +import org.apache.cloudstack.utils.security.SSLUtils; + import streamer.debug.MockServer; import streamer.debug.MockServer.Packet; import streamer.ssl.SSLState; @@ -139,7 +141,8 @@ public class SocketWrapperImpl extends PipelineImpl implements SocketWrapper { SSLSocketFactory sslSocketFactory = sslContext.getSocketFactory(); sslSocket = (SSLSocket)sslSocketFactory.createSocket(socket, address.getHostName(), address.getPort(), true); - sslSocket.setEnabledProtocols(new String[]{"TLSv1", "TLSv1.1", "TLSv1.2"}); + sslSocket.setEnabledProtocols(SSLUtils.getSupportedProtocols(sslSocket.getEnabledProtocols())); + sslSocket.startHandshake(); InputStream sis = sslSocket.getInputStream(); From 46301cbaf2367999103dcf0f6485fe00f48ec9dd Mon Sep 17 00:00:00 2001 From: Marcus Sorensen Date: Fri, 23 Jan 2015 02:34:08 -0700 Subject: [PATCH 111/391] CLOUDSTACK-8177 KVM: disable tickpolicy for kvmclock. Newer libvirt enforces that kvmclock cannot accept parameters. --- .../hypervisor/kvm/resource/LibvirtComputingResource.java | 2 +- .../kvm/resource/LibvirtComputingResourceTest.java | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 68c776ae248..691601fc8e8 100755 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -3723,7 +3723,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv clock.setClockOffset(ClockDef.ClockOffset.LOCALTIME); clock.setTimer("rtc", "catchup", null); } else if (vmTO.getType() != VirtualMachine.Type.User || isGuestPVEnabled(vmTO.getOs())) { - clock.setTimer("kvmclock", "catchup", null, _noKvmClock); + clock.setTimer("kvmclock", null, null, _noKvmClock); } vm.addComp(clock); diff --git a/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java b/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java index faa88f0cb65..a087f018c5a 100644 --- a/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java +++ b/plugins/hypervisors/kvm/test/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java @@ -94,7 +94,7 @@ public class LibvirtComputingResourceTest { vmStr += "b0f0a72d-7efb-3cad-a8ff-70ebf30b3af9\n"; vmStr += "" + os + "\n"; vmStr += "\n"; - vmStr += "\n"; + vmStr += "\n"; vmStr += "\n"; vmStr += "\n"; vmStr += "\n"; @@ -168,7 +168,7 @@ public class LibvirtComputingResourceTest { vmStr += "b0f0a72d-7efb-3cad-a8ff-70ebf30b3af9\n"; vmStr += "" + os + "\n"; vmStr += "\n"; - vmStr += "\n"; + vmStr += "\n"; vmStr += "\n"; vmStr += "\n"; vmStr += "\n"; @@ -240,7 +240,7 @@ public class LibvirtComputingResourceTest { vmStr += "b0f0a72d-7efb-3cad-a8ff-70ebf30b3af9\n"; vmStr += "" + os + "\n"; vmStr += "\n"; - vmStr += "\n"; + vmStr += "\n"; vmStr += "\n"; vmStr += "\n"; vmStr += "\n"; @@ -317,7 +317,7 @@ public class LibvirtComputingResourceTest { vmStr += "b0f0a72d-7efb-3cad-a8ff-70ebf30b3af9\n"; vmStr += "" + os + "\n"; vmStr += "\n"; - vmStr += "\n"; + vmStr += "\n"; vmStr += "\n"; vmStr += "\n"; vmStr += "\n"; From 0a8e73e8fe6a7077c90c55880d556f0f883fc83f Mon Sep 17 00:00:00 2001 From: Marcus Sorensen Date: Fri, 23 Jan 2015 02:44:08 -0700 Subject: [PATCH 112/391] LibvirtComputingResource: Expose KVMStoragePoolManager KVMStoragePoolManager is a singleton in practice, any plugin or extension of LibvirtComputingResource will need to act on the specific instance of KVMStoragePoolManager that LibvirtComputingResource has initialized. Therefore, expose this variable for those who wish to call storage commands from plugins or extensions. Conflicts: plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java --- .../hypervisor/kvm/resource/LibvirtComputingResource.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 691601fc8e8..f2dfc4ceb8b 100755 --- a/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -4075,6 +4075,10 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv return _storagePoolMgr.disconnectPhysicalDiskByPath(path); } + protected KVMStoragePoolManager getPoolManager() { + return _storagePoolMgr; + } + protected synchronized String attachOrDetachISO(Connect conn, String vmName, String isoPath, boolean isAttach) throws LibvirtException, URISyntaxException, InternalErrorException { String isoXml = null; From c44890f844a044723e16cac100e62f17011c55a5 Mon Sep 17 00:00:00 2001 From: Abhinandan Prateek Date: Fri, 23 Jan 2015 11:20:54 -0500 Subject: [PATCH 113/391] CLOUDSTACK-8178: fixed the mapping to match that on xen 6.5 --- setup/db/db/schema-442to450.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/db/db/schema-442to450.sql b/setup/db/db/schema-442to450.sql index 94984816fe4..798b2324a84 100644 --- a/setup/db/db/schema-442to450.sql +++ b/setup/db/db/schema-442to450.sql @@ -818,8 +818,8 @@ INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervis INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'Xenserver', '6.5.0', 'CentOS 7', 246, utc_timestamp(), 0); INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'Xenserver', '6.5.0', 'Debian Squeeze 6.0 (32-bit)', 132, utc_timestamp(), 0); INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'Xenserver', '6.5.0', 'Debian Squeeze 6.0 (64-bit)', 133, utc_timestamp(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'Xenserver', '6.5.0', 'Debian Wheezy 7 (32-bit)', 183, utc_timestamp(), 0); -INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'Xenserver', '6.5.0', 'Debian Wheezy 7 (64-bit)', 184, utc_timestamp(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'Xenserver', '6.5.0', 'Debian Wheezy 7.0 (32-bit)', 183, utc_timestamp(), 0); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'Xenserver', '6.5.0', 'Debian Wheezy 7.0 (64-bit)', 184, utc_timestamp(), 0); INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'Xenserver', '6.5.0', 'Oracle Enterprise Linux 5 (32-bit)', 16, utc_timestamp(), 0); INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'Xenserver', '6.5.0', 'Oracle Enterprise Linux 5 (64-bit)', 17, utc_timestamp(), 0); INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid,hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) VALUES (UUID(),'Xenserver', '6.5.0', 'Oracle Enterprise Linux 5 (32-bit)', 18, utc_timestamp(), 0); From d921163ea9536f69596083aab63fdb59512a8dd7 Mon Sep 17 00:00:00 2001 From: Anthony Xu Date: Fri, 23 Jan 2015 13:45:28 -0800 Subject: [PATCH 114/391] BUG-ID: CLOUDSTACK-8176 UI use xennetworklabel parameter name in zone create Wizards, in API, it uses xenservernetworklabel, the parameter name doesn't match, cause network name label not setup in CCP --- ui/scripts/zoneWizard.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/scripts/zoneWizard.js b/ui/scripts/zoneWizard.js index 66b34699eb0..299eff25a30 100755 --- a/ui/scripts/zoneWizard.js +++ b/ui/scripts/zoneWizard.js @@ -71,7 +71,7 @@ switch (hypervisor) { case 'XenServer': - hypervisorAttr = 'xennetworklabel'; + hypervisorAttr = 'xenservernetworklabel'; break; case 'KVM': hypervisorAttr = 'kvmnetworklabel'; From 766f8216a0bc96343a074c66757872052dd7b233 Mon Sep 17 00:00:00 2001 From: Daan Hoogland Date: Sat, 24 Jan 2015 13:29:18 +0100 Subject: [PATCH 115/391] state description c&p fix --- api/src/com/cloud/storage/Volume.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/com/cloud/storage/Volume.java b/api/src/com/cloud/storage/Volume.java index 91ad9551ec1..82b673b1bac 100755 --- a/api/src/com/cloud/storage/Volume.java +++ b/api/src/com/cloud/storage/Volume.java @@ -40,7 +40,7 @@ public interface Volume extends ControlledEntity, Identity, InternalIdentity, Ba Snapshotting("There is a snapshot created on this volume, not backed up to secondary storage yet"), Resizing("The volume is being resized"), Expunging("The volume is being expunging"), - Expunged("The volume is being expunging"), + Expunged("The volume has been expunged"), Destroy("The volume is destroyed, and can't be recovered."), Destroying("The volume is destroying, and can't be recovered."), UploadOp("The volume upload operation is in progress or in short the volume is on secondary storage"), From e474454c912af666eb062c821b6976dbf2e01cb6 Mon Sep 17 00:00:00 2001 From: Daan Hoogland Date: Mon, 26 Jan 2015 16:54:53 +0100 Subject: [PATCH 116/391] CLOUDSTACK-6485 prevent ipaddress asignment of gateway to gateway-interface on vpc router by setting vpcid to null in network --- engine/schema/src/com/cloud/network/dao/NetworkVO.java | 2 +- server/src/com/cloud/network/vpc/VpcManagerImpl.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/engine/schema/src/com/cloud/network/dao/NetworkVO.java b/engine/schema/src/com/cloud/network/dao/NetworkVO.java index 6d4add7c708..b0332106aec 100644 --- a/engine/schema/src/com/cloud/network/dao/NetworkVO.java +++ b/engine/schema/src/com/cloud/network/dao/NetworkVO.java @@ -606,7 +606,7 @@ public class NetworkVO implements Network { this.strechedL2Network = strechedL2Network; } - public void setVpcId(long vpcId) { + public void setVpcId(Long vpcId) { this.vpcId = vpcId; } } diff --git a/server/src/com/cloud/network/vpc/VpcManagerImpl.java b/server/src/com/cloud/network/vpc/VpcManagerImpl.java index c49da15d024..796b590cd05 100644 --- a/server/src/com/cloud/network/vpc/VpcManagerImpl.java +++ b/server/src/com/cloud/network/vpc/VpcManagerImpl.java @@ -1654,7 +1654,7 @@ public class VpcManagerImpl extends ManagerBase implements VpcManager, VpcProvis // A more permanent solution would be to define a type of 'gatewaynetwork' // so that handling code is not mixed between the two NetworkVO gatewaynet = _ntwkDao.findById(privateNtwk.getId()); - gatewaynet.setVpcId(vpcId); + gatewaynet.setVpcId(null); _ntwkDao.persist(gatewaynet); } From e9ce0df5d8a7173132b356040d036118a4612e39 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Mon, 15 Dec 2014 09:22:52 +0100 Subject: [PATCH 117/391] CLOUDSTACK-7316: usage server cannot start due to missing key in /etc/cloudstack/usage/ in Ubuntu setup (cherry picked from commit 6fb9746e5eeacadc10b7d0977b03959683058b33) Signed-off-by: Rohit Yadav --- debian/cloudstack-usage.postinst | 6 ++++++ .../schema/src/com/cloud/usage/dao/UsageVmDiskDaoImpl.java | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/debian/cloudstack-usage.postinst b/debian/cloudstack-usage.postinst index fa8650c53e6..8a475c44069 100644 --- a/debian/cloudstack-usage.postinst +++ b/debian/cloudstack-usage.postinst @@ -37,6 +37,12 @@ case "$1" in if [ -f "/etc/cloud/usage/log4j-cloud.xml" ]; then cp -a /etc/cloud/usage/log4j-cloud.xml /etc/cloudstack/usage/log4j-cloud.xml fi + + # Replacing key with management server key + if [ -f "/etc/cloudstack/management/key" ]; then + rm -rf /etc/cloudstack/usage/key + ln -s /etc/cloudstack/management/key /etc/cloudstack/usage/key + fi ;; esac diff --git a/engine/schema/src/com/cloud/usage/dao/UsageVmDiskDaoImpl.java b/engine/schema/src/com/cloud/usage/dao/UsageVmDiskDaoImpl.java index 3e759301315..4491c6730b9 100644 --- a/engine/schema/src/com/cloud/usage/dao/UsageVmDiskDaoImpl.java +++ b/engine/schema/src/com/cloud/usage/dao/UsageVmDiskDaoImpl.java @@ -35,7 +35,7 @@ import com.cloud.utils.exception.CloudRuntimeException; @Component @Local(value = {UsageVmDiskDao.class}) public class UsageVmDiskDaoImpl extends GenericDaoBase implements UsageVmDiskDao { - private static final Logger s_logger = Logger.getLogger(UsageVMInstanceDaoImpl.class.getName()); + private static final Logger s_logger = Logger.getLogger(UsageVmDiskDaoImpl.class.getName()); private static final String SELECT_LATEST_STATS = "SELECT uvd.account_id, uvd.zone_id, uvd.vm_id, uvd.volume_id, uvd.io_read, uvd.io_write, uvd.agg_io_read, uvd.agg_io_write, " + "uvd.bytes_read, uvd.bytes_write, uvd.agg_bytes_read, uvd.agg_bytes_write, uvd.event_time_millis " From eca528dec73bc8599089a44516c37894d56d1902 Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Wed, 28 Jan 2015 23:10:31 +0530 Subject: [PATCH 118/391] appliance: upgrade systemvm to latest packages Signed-off-by: Rohit Yadav --- tools/appliance/definitions/systemvm64template/postinstall.sh | 4 ++++ tools/appliance/definitions/systemvmtemplate/postinstall.sh | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/tools/appliance/definitions/systemvm64template/postinstall.sh b/tools/appliance/definitions/systemvm64template/postinstall.sh index b634f11d649..fc1333d1c8c 100644 --- a/tools/appliance/definitions/systemvm64template/postinstall.sh +++ b/tools/appliance/definitions/systemvm64template/postinstall.sh @@ -107,6 +107,10 @@ install_packages() { apt-get --no-install-recommends -q -y --force-yes install links:i386 libuuid1:i386 apt-get --no-install-recommends -q -y --force-yes install radvd + + # do a final update and upgrade + apt-get update + apt-get -y --force-yes } setup_accounts() { diff --git a/tools/appliance/definitions/systemvmtemplate/postinstall.sh b/tools/appliance/definitions/systemvmtemplate/postinstall.sh index a1ae79262f4..c08a9b1b02a 100644 --- a/tools/appliance/definitions/systemvmtemplate/postinstall.sh +++ b/tools/appliance/definitions/systemvmtemplate/postinstall.sh @@ -97,6 +97,10 @@ install_packages() { apt-get --no-install-recommends -q -y --force-yes install haproxy apt-get --no-install-recommends -q -y --force-yes install radvd + + # do a final update and upgrade + apt-get update + apt-get -y --force-yes } setup_accounts() { From a3fa59eaf197960ddfe985c273c79ade8d5eefe0 Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Wed, 28 Jan 2015 23:14:12 +0530 Subject: [PATCH 119/391] appliance: fix missing statement Signed-off-by: Rohit Yadav --- tools/appliance/definitions/systemvm64template/postinstall.sh | 2 +- tools/appliance/definitions/systemvmtemplate/postinstall.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/appliance/definitions/systemvm64template/postinstall.sh b/tools/appliance/definitions/systemvm64template/postinstall.sh index fc1333d1c8c..de632931fd3 100644 --- a/tools/appliance/definitions/systemvm64template/postinstall.sh +++ b/tools/appliance/definitions/systemvm64template/postinstall.sh @@ -110,7 +110,7 @@ install_packages() { # do a final update and upgrade apt-get update - apt-get -y --force-yes + apt-get -y --force-yes upgrade } setup_accounts() { diff --git a/tools/appliance/definitions/systemvmtemplate/postinstall.sh b/tools/appliance/definitions/systemvmtemplate/postinstall.sh index c08a9b1b02a..1ed12587837 100644 --- a/tools/appliance/definitions/systemvmtemplate/postinstall.sh +++ b/tools/appliance/definitions/systemvmtemplate/postinstall.sh @@ -100,7 +100,7 @@ install_packages() { # do a final update and upgrade apt-get update - apt-get -y --force-yes + apt-get -y --force-yes upgrade } setup_accounts() { From 805704721fbbbcce7aecfaa8aa214cfc8ab3bdfc Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Wed, 28 Jan 2015 23:21:47 +0530 Subject: [PATCH 120/391] appliance: hold on to openswan version, don't upgrade Signed-off-by: Rohit Yadav (cherry picked from commit a10a9d522c56f707a9674f1486f58697e8c5206a) Signed-off-by: Rohit Yadav --- tools/appliance/definitions/systemvm64template/postinstall.sh | 1 + tools/appliance/definitions/systemvmtemplate/postinstall.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/tools/appliance/definitions/systemvm64template/postinstall.sh b/tools/appliance/definitions/systemvm64template/postinstall.sh index de632931fd3..134571c9662 100644 --- a/tools/appliance/definitions/systemvm64template/postinstall.sh +++ b/tools/appliance/definitions/systemvm64template/postinstall.sh @@ -60,6 +60,7 @@ install_packages() { echo "openswan openswan/install_x509_certificate boolean false" | debconf-set-selections echo "openswan openswan/install_x509_certificate seen true" | debconf-set-selections apt-get --no-install-recommends -q -y --force-yes install openswan=1:2.6.37-3 + apt-mark hold openswan # xenstore utils apt-get --no-install-recommends -q -y --force-yes install xenstore-utils libxenstore3.0 diff --git a/tools/appliance/definitions/systemvmtemplate/postinstall.sh b/tools/appliance/definitions/systemvmtemplate/postinstall.sh index 1ed12587837..eb4c5e53f28 100644 --- a/tools/appliance/definitions/systemvmtemplate/postinstall.sh +++ b/tools/appliance/definitions/systemvmtemplate/postinstall.sh @@ -60,6 +60,7 @@ install_packages() { echo "openswan openswan/install_x509_certificate boolean false" | debconf-set-selections echo "openswan openswan/install_x509_certificate seen true" | debconf-set-selections apt-get --no-install-recommends -q -y --force-yes install openswan=1:2.6.37-3 + apt-mark hold openswan # xenstore utils apt-get --no-install-recommends -q -y --force-yes install xenstore-utils libxenstore3.0 From 1bab1d0855f8813a54028aa58715db75d9c06ca0 Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Wed, 28 Jan 2015 13:41:00 +0530 Subject: [PATCH 121/391] use a preferable protocol that works on jvm 1.6 Signed-off-by: Rohit Yadav (cherry picked from commit f5f6c2d1a74444bbbf5a5f4565953b6410b87a5c) Signed-off-by: Rohit Yadav Conflicts: services/console-proxy-rdp/rdpconsole/src/main/java/streamer/SocketWrapperImpl.java --- .../org/apache/cloudstack/mom/rabbitmq/RabbitMQEventBus.java | 2 +- .../rdpconsole/src/main/java/streamer/SocketWrapperImpl.java | 4 +--- utils/src/org/apache/cloudstack/utils/security/SSLUtils.java | 4 ++-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/plugins/event-bus/rabbitmq/src/org/apache/cloudstack/mom/rabbitmq/RabbitMQEventBus.java b/plugins/event-bus/rabbitmq/src/org/apache/cloudstack/mom/rabbitmq/RabbitMQEventBus.java index 25ecb75b958..bd414dbf4bc 100644 --- a/plugins/event-bus/rabbitmq/src/org/apache/cloudstack/mom/rabbitmq/RabbitMQEventBus.java +++ b/plugins/event-bus/rabbitmq/src/org/apache/cloudstack/mom/rabbitmq/RabbitMQEventBus.java @@ -59,7 +59,7 @@ public class RabbitMQEventBus extends ManagerBase implements EventBus { private static Integer port; private static String username; private static String password; - private static String secureProtocol = "TLSv1.2"; + private static String secureProtocol = "TLSv1"; public synchronized static void setVirtualHost(String virtualHost) { RabbitMQEventBus.virtualHost = virtualHost; diff --git a/services/console-proxy-rdp/rdpconsole/src/main/java/streamer/SocketWrapperImpl.java b/services/console-proxy-rdp/rdpconsole/src/main/java/streamer/SocketWrapperImpl.java index 6d996286a36..14089ce609c 100755 --- a/services/console-proxy-rdp/rdpconsole/src/main/java/streamer/SocketWrapperImpl.java +++ b/services/console-proxy-rdp/rdpconsole/src/main/java/streamer/SocketWrapperImpl.java @@ -48,8 +48,6 @@ public class SocketWrapperImpl extends PipelineImpl implements SocketWrapper { protected SSLSocket sslSocket; - protected String sslVersionToUse = "TLSv1.2"; - protected SSLState sslState; public SocketWrapperImpl(String id, SSLState sslState) { @@ -134,7 +132,7 @@ public class SocketWrapperImpl extends PipelineImpl implements SocketWrapper { // Use most secure implementation of SSL available now. // JVM will try to negotiate TLS1.2, then will fallback to TLS1.0, if // TLS1.2 is not supported. - SSLContext sslContext = SSLContext.getInstance(sslVersionToUse); + SSLContext sslContext = SSLUtils.getSSLContext(); // Trust all certificates (FIXME: insecure) sslContext.init(null, new TrustManager[] {new TrustAllX509TrustManager(sslState)}, null); diff --git a/utils/src/org/apache/cloudstack/utils/security/SSLUtils.java b/utils/src/org/apache/cloudstack/utils/security/SSLUtils.java index 7f9ee77c985..3de4c50c7bf 100644 --- a/utils/src/org/apache/cloudstack/utils/security/SSLUtils.java +++ b/utils/src/org/apache/cloudstack/utils/security/SSLUtils.java @@ -42,10 +42,10 @@ public class SSLUtils { } public static SSLContext getSSLContext() throws NoSuchAlgorithmException { - return SSLContext.getInstance("TLSv1.2"); + return SSLContext.getInstance("TLSv1"); } public static SSLContext getSSLContext(String provider) throws NoSuchAlgorithmException, NoSuchProviderException { - return SSLContext.getInstance("TLSv1.2", provider); + return SSLContext.getInstance("TLSv1", provider); } } From 737edd90dc253789e442634b389ca5db835ee6e0 Mon Sep 17 00:00:00 2001 From: Santhosh Edukulla Date: Thu, 9 Oct 2014 11:42:49 +0530 Subject: [PATCH 122/391] Fixed few coverity patches NPE in delete firewall rules observed, cherry-picking fix from master. (cherry picked from commit 31a42d2b7a5a9d3dbf10dc680d7e8877ed4e40c6) Signed-off-by: Rohit Yadav --- .../network/UpdatePhysicalNetworkCmd.java | 8 +- .../user/firewall/DeleteFirewallRuleCmd.java | 5 +- .../command/user/vpn/UpdateVpnGatewayCmd.java | 6 +- .../manager/ClusteredAgentManagerImpl.java | 43 +++--- .../com/cloud/utils/db/GenericDaoBase.java | 122 ++++++++---------- .../discovery/ApiDiscoveryTest.java | 24 ++-- .../network/element/NuageVspElement.java | 6 +- utils/src/com/cloud/utils/script/Script.java | 24 ++-- 8 files changed, 121 insertions(+), 117 deletions(-) diff --git a/api/src/org/apache/cloudstack/api/command/admin/network/UpdatePhysicalNetworkCmd.java b/api/src/org/apache/cloudstack/api/command/admin/network/UpdatePhysicalNetworkCmd.java index 8e29c393813..ed3fa97ce7a 100644 --- a/api/src/org/apache/cloudstack/api/command/admin/network/UpdatePhysicalNetworkCmd.java +++ b/api/src/org/apache/cloudstack/api/command/admin/network/UpdatePhysicalNetworkCmd.java @@ -97,9 +97,11 @@ public class UpdatePhysicalNetworkCmd extends BaseAsyncCmd { @Override public void execute() { PhysicalNetwork result = _networkService.updatePhysicalNetwork(getId(), getNetworkSpeed(), getTags(), getVlan(), getState()); - PhysicalNetworkResponse response = _responseGenerator.createPhysicalNetworkResponse(result); - response.setResponseName(getCommandName()); - this.setResponseObject(response); + if (result != null) { + PhysicalNetworkResponse response = _responseGenerator.createPhysicalNetworkResponse(result); + response.setResponseName(getCommandName()); + this.setResponseObject(response); + } } @Override diff --git a/api/src/org/apache/cloudstack/api/command/user/firewall/DeleteFirewallRuleCmd.java b/api/src/org/apache/cloudstack/api/command/user/firewall/DeleteFirewallRuleCmd.java index f8d1c18329a..0f100124610 100644 --- a/api/src/org/apache/cloudstack/api/command/user/firewall/DeleteFirewallRuleCmd.java +++ b/api/src/org/apache/cloudstack/api/command/user/firewall/DeleteFirewallRuleCmd.java @@ -113,7 +113,10 @@ public class DeleteFirewallRuleCmd extends BaseAsyncCmd { @Override public Long getSyncObjId() { - return _firewallService.getFirewallRule(id).getNetworkId(); + FirewallRule fwlrule = _firewallService.getFirewallRule(id); + if (fwlrule != null) + return fwlrule.getNetworkId(); + return null; } @Override diff --git a/api/src/org/apache/cloudstack/api/command/user/vpn/UpdateVpnGatewayCmd.java b/api/src/org/apache/cloudstack/api/command/user/vpn/UpdateVpnGatewayCmd.java index 66c59ad9f1b..bbe820a83c4 100644 --- a/api/src/org/apache/cloudstack/api/command/user/vpn/UpdateVpnGatewayCmd.java +++ b/api/src/org/apache/cloudstack/api/command/user/vpn/UpdateVpnGatewayCmd.java @@ -85,8 +85,10 @@ public class UpdateVpnGatewayCmd extends BaseAsyncCustomIdCmd { @Override public void execute() { Site2SiteVpnGateway result = _s2sVpnService.updateVpnGateway(id, this.getCustomId(), getDisplay()); - Site2SiteVpnGatewayResponse response = _responseGenerator.createSite2SiteVpnGatewayResponse(result); - response.setResponseName(getCommandName()); + if (result != null) { + Site2SiteVpnGatewayResponse response = _responseGenerator.createSite2SiteVpnGatewayResponse(result); + response.setResponseName(getCommandName()); + } } @Override diff --git a/engine/orchestration/src/com/cloud/agent/manager/ClusteredAgentManagerImpl.java b/engine/orchestration/src/com/cloud/agent/manager/ClusteredAgentManagerImpl.java index 72d7c3b8ba9..600dca262fb 100755 --- a/engine/orchestration/src/com/cloud/agent/manager/ClusteredAgentManagerImpl.java +++ b/engine/orchestration/src/com/cloud/agent/manager/ClusteredAgentManagerImpl.java @@ -195,35 +195,36 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust long cutSeconds = (System.currentTimeMillis() >> 10) - getTimeout(); List hosts = _hostDao.findAndUpdateDirectAgentToLoad(cutSeconds, LoadSize.value().longValue(), _nodeId); List appliances = _hostDao.findAndUpdateApplianceToLoad(cutSeconds, _nodeId); - hosts.addAll(appliances); - if (hosts != null && hosts.size() > 0) { - s_logger.debug("Found " + hosts.size() + " unmanaged direct hosts, processing connect for them..."); - for (HostVO host : hosts) { - try { - AgentAttache agentattache = findAttache(host.getId()); - if (agentattache != null) { - // already loaded, skip - if (agentattache.forForward()) { - if (s_logger.isInfoEnabled()) { - s_logger.info(host + " is detected down, but we have a forward attache running, disconnect this one before launching the host"); + if (hosts != null) { + hosts.addAll(appliances); + if (hosts.size() > 0) { + s_logger.debug("Found " + hosts.size() + " unmanaged direct hosts, processing connect for them..."); + for (HostVO host : hosts) { + try { + AgentAttache agentattache = findAttache(host.getId()); + if (agentattache != null) { + // already loaded, skip + if (agentattache.forForward()) { + if (s_logger.isInfoEnabled()) { + s_logger.info(host + " is detected down, but we have a forward attache running, disconnect this one before launching the host"); + } + removeAgent(agentattache, Status.Disconnected); + } else { + continue; } - removeAgent(agentattache, Status.Disconnected); - } else { - continue; } - } - if (s_logger.isDebugEnabled()) { - s_logger.debug("Loading directly connected host " + host.getId() + "(" + host.getName() + ")"); + if (s_logger.isDebugEnabled()) { + s_logger.debug("Loading directly connected host " + host.getId() + "(" + host.getName() + ")"); + } + loadDirectlyConnectedHost(host, false); + } catch (Throwable e) { + s_logger.warn(" can not load directly connected host " + host.getId() + "(" + host.getName() + ") due to ", e); } - loadDirectlyConnectedHost(host, false); - } catch (Throwable e) { - s_logger.warn(" can not load directly connected host " + host.getId() + "(" + host.getName() + ") due to ", e); } } } - if (s_logger.isTraceEnabled()) { s_logger.trace("End scanning directly connected hosts"); } diff --git a/framework/db/src/com/cloud/utils/db/GenericDaoBase.java b/framework/db/src/com/cloud/utils/db/GenericDaoBase.java index 4c474044380..e75646a79fa 100755 --- a/framework/db/src/com/cloud/utils/db/GenericDaoBase.java +++ b/framework/db/src/com/cloud/utils/db/GenericDaoBase.java @@ -1641,81 +1641,71 @@ public abstract class GenericDaoBase extends Compone @SuppressWarnings("unchecked") protected void loadCollection(T entity, Attribute attr) { EcInfo ec = (EcInfo)attr.attache; - TransactionLegacy txn = TransactionLegacy.currentTxn(); - ResultSet rs = null; - PreparedStatement pstmt = null; - try { - pstmt = txn.prepareStatement(ec.selectSql); + try(PreparedStatement pstmt = txn.prepareStatement(ec.selectSql);) + { pstmt.setObject(1, _idField.get(entity)); - rs = pstmt.executeQuery(); - ArrayList lst = new ArrayList(); - if (ec.targetClass == Integer.class) { - while (rs.next()) { - lst.add(rs.getInt(1)); + try(ResultSet rs = pstmt.executeQuery();) + { + ArrayList lst = new ArrayList(); + if (ec.targetClass == Integer.class) { + while (rs.next()) { + lst.add(rs.getInt(1)); + } + } else if (ec.targetClass == Long.class) { + while (rs.next()) { + lst.add(rs.getLong(1)); + } + } else if (ec.targetClass == String.class) { + while (rs.next()) { + lst.add(rs.getString(1)); + } + } else if (ec.targetClass == Short.class) { + while (rs.next()) { + lst.add(rs.getShort(1)); + } + } else if (ec.targetClass == Date.class) { + while (rs.next()) { + lst.add(DateUtil.parseDateString(s_gmtTimeZone, rs.getString(1))); + } + } else if (ec.targetClass == Boolean.class) { + while (rs.next()) { + lst.add(rs.getBoolean(1)); + } + } else { + assert (false) : "You'll need to add more classeses"; } - } else if (ec.targetClass == Long.class) { - while (rs.next()) { - lst.add(rs.getLong(1)); + if (ec.rawClass == null) { + Object[] array = (Object[]) Array.newInstance(ec.targetClass); + lst.toArray(array); + try { + attr.field.set(entity, array); + } catch (IllegalArgumentException e) { + throw new CloudRuntimeException("Come on we screen for this stuff, don't we?", e); + } catch (IllegalAccessException e) { + throw new CloudRuntimeException("Come on we screen for this stuff, don't we?", e); + } + } else { + try { + Collection coll = (Collection) ec.rawClass.newInstance(); + coll.addAll(lst); + attr.field.set(entity, coll); + } catch (IllegalAccessException e) { + throw new CloudRuntimeException("Come on we screen for this stuff, don't we?", e); + } catch (InstantiationException e) { + throw new CloudRuntimeException("Never should happen", e); + } } - } else if (ec.targetClass == String.class) { - while (rs.next()) { - lst.add(rs.getString(1)); - } - } else if (ec.targetClass == Short.class) { - while (rs.next()) { - lst.add(rs.getShort(1)); - } - } else if (ec.targetClass == Date.class) { - while (rs.next()) { - lst.add(DateUtil.parseDateString(s_gmtTimeZone, rs.getString(1))); - } - } else if (ec.targetClass == Boolean.class) { - while (rs.next()) { - lst.add(rs.getBoolean(1)); - } - } else { - assert (false) : "You'll need to add more classeses"; } - - if (ec.rawClass == null) { - Object[] array = (Object[])Array.newInstance(ec.targetClass); - lst.toArray(array); - try { - attr.field.set(entity, array); - } catch (IllegalArgumentException e) { - throw new CloudRuntimeException("Come on we screen for this stuff, don't we?", e); - } catch (IllegalAccessException e) { - throw new CloudRuntimeException("Come on we screen for this stuff, don't we?", e); - } - } else { - try { - Collection coll = (Collection)ec.rawClass.newInstance(); - coll.addAll(lst); - attr.field.set(entity, coll); - } catch (IllegalAccessException e) { - throw new CloudRuntimeException("Come on we screen for this stuff, don't we?", e); - } catch (InstantiationException e) { - throw new CloudRuntimeException("Never should happen", e); - } + catch (SQLException e) { + throw new CloudRuntimeException("loadCollection: Exception : " +e.getMessage(), e); } } catch (SQLException e) { - throw new CloudRuntimeException("Error executing " + pstmt, e); + throw new CloudRuntimeException("loadCollection: Exception : " +e.getMessage(), e); } catch (IllegalArgumentException e) { - throw new CloudRuntimeException("Error executing " + pstmt, e); + throw new CloudRuntimeException("loadCollection: Exception : " +e.getMessage(), e); } catch (IllegalAccessException e) { - throw new CloudRuntimeException("Error executing " + pstmt, e); - } finally { - try { - if (rs != null) { - rs.close(); - } - if (pstmt != null) { - pstmt.close(); - } - } catch (SQLException e) { - s_logger.error("Why are we getting an exception at close? ", e); - } + throw new CloudRuntimeException("loadCollection: Exception : " +e.getMessage(), e); } } diff --git a/plugins/api/discovery/test/org/apache/cloudstack/discovery/ApiDiscoveryTest.java b/plugins/api/discovery/test/org/apache/cloudstack/discovery/ApiDiscoveryTest.java index 30cb0bc87ae..49bf5a55dc5 100644 --- a/plugins/api/discovery/test/org/apache/cloudstack/discovery/ApiDiscoveryTest.java +++ b/plugins/api/discovery/test/org/apache/cloudstack/discovery/ApiDiscoveryTest.java @@ -82,21 +82,25 @@ public class ApiDiscoveryTest { @Test public void verifyListSingleApi() throws Exception { ListResponse responses = (ListResponse)s_discoveryService.listApis(testUser, testApiName); - ApiDiscoveryResponse response = responses.getResponses().get(0); - assertTrue("No. of response items should be one", responses.getCount() == 1); - assertEquals("Error in api name", testApiName, response.getName()); - assertEquals("Error in api description", testApiDescription, response.getDescription()); - assertEquals("Error in api since", testApiSince, response.getSince()); - assertEquals("Error in api isAsync", testApiAsync, response.getAsync()); + if (responses != null) { + ApiDiscoveryResponse response = responses.getResponses().get(0); + assertTrue("No. of response items should be one", responses.getCount() == 1); + assertEquals("Error in api name", testApiName, response.getName()); + assertEquals("Error in api description", testApiDescription, response.getDescription()); + assertEquals("Error in api since", testApiSince, response.getSince()); + assertEquals("Error in api isAsync", testApiAsync, response.getAsync()); + } } @Test public void verifyListApis() throws Exception { ListResponse responses = (ListResponse)s_discoveryService.listApis(testUser, null); - assertTrue("No. of response items > 1", responses.getCount() == 1); - for (ApiDiscoveryResponse response : responses.getResponses()) { - assertFalse("API name is empty", response.getName().isEmpty()); - assertFalse("API description is empty", response.getDescription().isEmpty()); + if (responses != null) { + assertTrue("No. of response items > 1", responses.getCount().intValue() == 1); + for (ApiDiscoveryResponse response : responses.getResponses()) { + assertFalse("API name is empty", response.getName().isEmpty()); + assertFalse("API description is empty", response.getDescription().isEmpty()); + } } } } diff --git a/plugins/network-elements/nuage-vsp/src/com/cloud/network/element/NuageVspElement.java b/plugins/network-elements/nuage-vsp/src/com/cloud/network/element/NuageVspElement.java index 06f9733fc0e..e777268e424 100644 --- a/plugins/network-elements/nuage-vsp/src/com/cloud/network/element/NuageVspElement.java +++ b/plugins/network-elements/nuage-vsp/src/com/cloud/network/element/NuageVspElement.java @@ -405,13 +405,15 @@ public class NuageVspElement extends AdapterBase implements ConnectivityProvider @Override public boolean applyNetworkACLs(Network network, List rules) throws ResourceUnavailableException { - s_logger.debug("Handling applyNetworkACLs for network " + network.getName() + " with " + rules.size() + " Network ACLs"); if (rules == null || rules.isEmpty()) { s_logger.debug("No rules to apply. So, delete all the existing ACL in VSP from Subnet with uuid " + network.getUuid()); } else { s_logger.debug("New rules has to applied. So, delete all the existing ACL in VSP from Subnet with uuid " + network.getUuid()); } - applyACLRules(network, rules, true); + if (rules != null) { + s_logger.debug("Handling applyNetworkACLs for network " + network.getName() + " with " + rules.size() + " Network ACLs"); + applyACLRules(network, rules, true); + } return true; } diff --git a/utils/src/com/cloud/utils/script/Script.java b/utils/src/com/cloud/utils/script/Script.java index 9713b144851..49734ae808a 100755 --- a/utils/src/com/cloud/utils/script/Script.java +++ b/utils/src/com/cloud/utils/script/Script.java @@ -313,19 +313,19 @@ public class Script implements Callable { @Override public void run() { - done = false; - try { - result = interpreter.interpret(reader); - } catch (IOException ex) { - result = stackTraceAsString(ex); - } catch (Exception ex) { - result = stackTraceAsString(ex); - } finally { - synchronized (this) { - done = true; - notifyAll(); + synchronized(this) { + done = false; + try { + result = interpreter.interpret(reader); + } catch (IOException ex) { + result = stackTraceAsString(ex); + } catch (Exception ex) { + result = stackTraceAsString(ex); + } finally { + done = true; + notifyAll(); + IOUtils.closeQuietly(reader); } - IOUtils.closeQuietly(reader); } } From 8f4aa23ae03e1fccb4357e95fbc71134616caeab Mon Sep 17 00:00:00 2001 From: Likitha Shetty Date: Mon, 15 Dec 2014 15:58:23 +0530 Subject: [PATCH 123/391] CLOUDSTACK-8127. VM name not displayed after attaching data disk to a VM. If user hasn't supplied a display name for a VM, default it to VM name in listVolume response. This behaviour is identical to listVirtualMachine response. (cherry picked from commit ff7997a2b139ed25f301ed1b8ad388864fcb9980) Signed-off-by: Rohit Yadav --- server/src/com/cloud/api/query/dao/VolumeJoinDaoImpl.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/server/src/com/cloud/api/query/dao/VolumeJoinDaoImpl.java b/server/src/com/cloud/api/query/dao/VolumeJoinDaoImpl.java index 97a1ff27cb0..68a578fd218 100644 --- a/server/src/com/cloud/api/query/dao/VolumeJoinDaoImpl.java +++ b/server/src/com/cloud/api/query/dao/VolumeJoinDaoImpl.java @@ -91,7 +91,11 @@ public class VolumeJoinDaoImpl extends GenericDaoBase implem volResponse.setVirtualMachineId(volume.getVmUuid()); volResponse.setVirtualMachineName(volume.getVmName()); volResponse.setVirtualMachineState(volume.getVmState().toString()); - volResponse.setVirtualMachineDisplayName(volume.getVmDisplayName()); + if (volume.getVmDisplayName() != null) { + volResponse.setVirtualMachineDisplayName(volume.getVmDisplayName()); + } else { + volResponse.setVirtualMachineDisplayName(volume.getVmName()); + } } volResponse.setProvisioningType(volume.getProvisioningType().toString()); From c8a437b782296e2b4a9727b1204d7f1693f2c806 Mon Sep 17 00:00:00 2001 From: Likitha Shetty Date: Tue, 28 Oct 2014 16:38:18 +0530 Subject: [PATCH 124/391] BUG-ID: CLOUDSTACK-7102. Volume migration fails with 'VM i-2-3-VM does not exist in VMware datacenter' expection. Look for a VM in vCenter based on both the vCenter name and CS internal name (required in case 'vm.instancename.flag' is enabled). During Attach Volume and Volume Migration, for lookup and other operations use VM's name as obtained from vCenter instead of using the name set in the agent command. (cherry picked from commit b8fdda0a34f5338db284ee616c5e04206c97b82e) Signed-off-by: Rohit Yadav --- .../vmware/resource/VmwareResource.java | 6 ++++-- .../resource/VmwareStorageProcessor.java | 1 + .../hypervisor/vmware/mo/DatacenterMO.java | 18 ++++++------------ 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java index f55ca67cb40..3b339ab6af6 100755 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -2921,7 +2921,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } VirtualMachineTO vmTo = cmd.getVirtualMachine(); - final String vmName = vmTo.getName(); + String vmName = vmTo.getName(); VmwareHypervisorHost srcHyperHost = null; VmwareHypervisorHost tgtHyperHost = null; @@ -2971,6 +2971,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa s_logger.error(msg); throw new Exception(msg); } + vmName = vmMo.getName(); // Get details of each target datastore & attach to source host. for (Entry entry : volToFiler.entrySet()) { @@ -3100,7 +3101,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa s_logger.info("Executing resource MigrateVolumeCommand: " + _gson.toJson(cmd)); } - final String vmName = cmd.getAttachedVmName(); + String vmName = cmd.getAttachedVmName(); VirtualMachineMO vmMo = null; VmwareHypervisorHost srcHyperHost = null; @@ -3127,6 +3128,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa s_logger.error(msg); throw new Exception(msg); } + vmName = vmMo.getName(); morDs = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(srcHyperHost, tgtDsName); if (morDs == null) { String msg = "Unable to find the mounted datastore with name " + tgtDsName + " to execute MigrateVolumeCommand"; diff --git a/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageProcessor.java index 757826154d1..621f005bd57 100644 --- a/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -1290,6 +1290,7 @@ public class VmwareStorageProcessor implements StorageProcessor { s_logger.error(msg); throw new Exception(msg); } + vmName = vmMo.getName(); ManagedObjectReference morDs = null; diff --git a/vmware-base/src/com/cloud/hypervisor/vmware/mo/DatacenterMO.java b/vmware-base/src/com/cloud/hypervisor/vmware/mo/DatacenterMO.java index ef137836dd2..39a30beadfb 100755 --- a/vmware-base/src/com/cloud/hypervisor/vmware/mo/DatacenterMO.java +++ b/vmware-base/src/com/cloud/hypervisor/vmware/mo/DatacenterMO.java @@ -82,19 +82,13 @@ public class DatacenterMO extends BaseMO { } public VirtualMachineMO findVm(String vmName) throws Exception { - List ocs = getVmPropertiesOnDatacenterVmFolder(new String[] {"name"}); - if (ocs != null && ocs.size() > 0) { - for (ObjectContent oc : ocs) { - List props = oc.getPropSet(); - if (props != null) { - for (DynamicProperty prop : props) { - if (prop.getVal().toString().equals(vmName)) - return new VirtualMachineMO(_context, oc.getObj()); - } - } - } + int key = getCustomFieldKey("VirtualMachine", CustomFieldConstants.CLOUD_VM_INTERNAL_NAME); + if (key == 0) { + s_logger.warn("Custom field " + CustomFieldConstants.CLOUD_VM_INTERNAL_NAME + " is not registered ?!"); } - return null; + String instanceNameCustomField = "value[" + key + "]"; + List ocs = getVmPropertiesOnDatacenterVmFolder(new String[] {"name", instanceNameCustomField}); + return HypervisorHostHelper.findVmFromObjectContent(_context, ocs.toArray(new ObjectContent[0]), vmName, instanceNameCustomField); } public List findVmByNameAndLabel(String vmLabel) throws Exception { From ba00a1a08109dd1596bbb37c2b5743349c4a5a55 Mon Sep 17 00:00:00 2001 From: Sateesh Chodapuneedi Date: Tue, 9 Dec 2014 00:47:14 +0530 Subject: [PATCH 125/391] CLOUDSTACK-7803 Storage live migration of instance may not happen if instance has ISO attached. BUG-ID:CLOUDSTACK-7803 Summary:Ensure secondary storage is mounted on target host. Reviewed-By:Rajesh Signed-off-by: Sateesh Chodapuneedi (cherry picked from commit bca015306e5c16915b5a76d846294c593e1df15c) Signed-off-by: Rohit Yadav --- .../hypervisor/vmware/resource/VmwareResource.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 3b339ab6af6..3b91f824e68 100755 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -3031,7 +3031,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa throw new Exception(msg); } mgr.prepareSecondaryStorageStore(secStoreUrl); - ManagedObjectReference morSecDs = prepareSecondaryDatastoreOnHost(secStoreUrl); + ManagedObjectReference morSecDs = prepareSecondaryDatastoreOnSpecificHost(secStoreUrl, tgtHyperHost); if (morSecDs == null) { String msg = "Failed to prepare secondary storage on host, secondary store url: " + secStoreUrl; throw new Exception(msg); @@ -3384,6 +3384,18 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa return morDatastore; } + public synchronized ManagedObjectReference prepareSecondaryDatastoreOnSpecificHost(String storeUrl, VmwareHypervisorHost hyperHost) throws Exception { + String storeName = getSecondaryDatastoreUUID(storeUrl); + URI uri = new URI(storeUrl); + + ManagedObjectReference morDatastore = hyperHost.mountDatastore(false, uri.getHost(), 0, uri.getPath(), storeName.replace("-", "")); + + if (morDatastore == null) + throw new Exception("Unable to mount secondary storage on host. storeUrl: " + storeUrl); + + return morDatastore; + } + private static String getSecondaryDatastoreUUID(String storeUrl) { return UUID.nameUUIDFromBytes(storeUrl.getBytes()).toString(); } From b2fa91629e3c07c6e05b9d0c52370cf02142fc38 Mon Sep 17 00:00:00 2001 From: Likitha Shetty Date: Fri, 31 Oct 2014 16:26:29 +0530 Subject: [PATCH 126/391] CLOUDSTACK-8110. VM name in vCenter should be VM's DisplayName if global config 'vm.instancename.flag' is set. (cherry picked from commit 51ee90074d5b53be261dcc746e4bd8a1058dfb6a) Signed-off-by: Rohit Yadav --- .../cloud/hypervisor/vmware/resource/VmwareResource.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 3b91f824e68..4e71090e4db 100755 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -1834,10 +1834,8 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa private Pair composeVmNames(VirtualMachineTO vmSpec) { String vmInternalCSName = vmSpec.getName(); String vmNameOnVcenter = vmSpec.getName(); - if (vmSpec.getType() == VirtualMachine.Type.User && _instanceNameFlag && vmSpec.getHostName() != null) { - String[] tokens = vmInternalCSName.split("-"); - assert (tokens.length >= 3); // vmInternalCSName has format i-x-y- - vmNameOnVcenter = String.format("%s-%s-%s-%s", tokens[0], tokens[1], tokens[2], vmSpec.getHostName()); + if (_instanceNameFlag && vmSpec.getHostName() != null) { + vmNameOnVcenter = vmSpec.getHostName(); } return new Pair(vmInternalCSName, vmNameOnVcenter); } From 45d32234a69b9e20372d00cc3d6d8cd48f4ea89c Mon Sep 17 00:00:00 2001 From: Likitha Shetty Date: Mon, 10 Nov 2014 10:58:11 +0530 Subject: [PATCH 127/391] CLOUDSTACK-8112. CS allows creation of VM's with the same Display name when vm.instancename.flag is set to true. Before registering a VM check if a different CS VM with same name exists in vCenter. (cherry picked from commit 33179cce56b15f0632e38afa260cb829bb2a2273) Signed-off-by: Rohit Yadav --- .../cloud/vm/VirtualMachineManagerImpl.java | 3 ++ .../vmware/resource/VmwareResource.java | 19 +++++++++-- .../hypervisor/vmware/mo/DatacenterMO.java | 32 +++++++++++++++++++ 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java index 0051cff843f..8c7388d342a 100755 --- a/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java @@ -1067,6 +1067,9 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } } s_logger.info("Unable to start VM on " + dest.getHost() + " due to " + (startAnswer == null ? " no start answer" : startAnswer.getDetails())); + if (startAnswer.getContextParam("stopRetry") != null) { + break; + } } catch (OperationTimedoutException e) { s_logger.debug("Unable to send the start command to host " + dest.getHost()); diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 4e71090e4db..9ad3a5f3067 100755 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -1324,6 +1324,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } VirtualMachineTO vmSpec = cmd.getVirtualMachine(); + boolean vmAlreadyExistsInVcenter = false; Pair names = composeVmNames(vmSpec); String vmInternalCSName = names.first(); @@ -1335,6 +1336,17 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa VmwareManager mgr = context.getStockObject(VmwareManager.CONTEXT_STOCK_NAME); VmwareHypervisorHost hyperHost = getHyperHost(context); + DatacenterMO dcMo = new DatacenterMO(hyperHost.getContext(), hyperHost.getHyperHostDatacenter()); + + // Validate VM name is unique in Datacenter + VirtualMachineMO vmInVcenter = dcMo.checkIfVmAlreadyExistsInVcenter(vmNameOnVcenter, vmInternalCSName); + if(vmInVcenter != null) { + vmAlreadyExistsInVcenter = true; + String msg = "VM with name: " + vmNameOnVcenter +" already exists in vCenter."; + s_logger.error(msg); + throw new Exception(msg); + } + DiskTO[] disks = validateDisks(vmSpec.getDisks()); assert (disks.length > 0); NicTO[] nics = vmSpec.getNics(); @@ -1353,7 +1365,6 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa throw new Exception(msg); } - DatacenterMO dcMo = new DatacenterMO(hyperHost.getContext(), hyperHost.getHyperHostDatacenter()); VirtualMachineDiskInfoBuilder diskInfoBuilder = null; VirtualMachineMO vmMo = hyperHost.findVmOnHyperHost(vmInternalCSName); boolean hasSnapshot = false; @@ -1738,7 +1749,11 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa String msg = "StartCommand failed due to " + VmwareHelper.getExceptionMessage(e); s_logger.warn(msg, e); - return new StartAnswer(cmd, msg); + StartAnswer startAnswer = new StartAnswer(cmd, msg); + if(vmAlreadyExistsInVcenter) { + startAnswer.setContextParam("stopRetry", "true"); + } + return startAnswer; } finally { } } diff --git a/vmware-base/src/com/cloud/hypervisor/vmware/mo/DatacenterMO.java b/vmware-base/src/com/cloud/hypervisor/vmware/mo/DatacenterMO.java index 39a30beadfb..38b1565ca20 100755 --- a/vmware-base/src/com/cloud/hypervisor/vmware/mo/DatacenterMO.java +++ b/vmware-base/src/com/cloud/hypervisor/vmware/mo/DatacenterMO.java @@ -125,6 +125,38 @@ public class DatacenterMO extends BaseMO { return list; } + public VirtualMachineMO checkIfVmAlreadyExistsInVcenter(String vmNameOnVcenter, String vmNameInCS) throws Exception { + int key = getCustomFieldKey("VirtualMachine", CustomFieldConstants.CLOUD_VM_INTERNAL_NAME); + if (key == 0) { + s_logger.warn("Custom field " + CustomFieldConstants.CLOUD_VM_INTERNAL_NAME + " is not registered ?!"); + } + + List ocs = getVmPropertiesOnDatacenterVmFolder(new String[] {"name", String.format("value[%d]", key)}); + if (ocs != null && ocs.size() > 0) { + for (ObjectContent oc : ocs) { + List props = oc.getPropSet(); + if (props != null) { + String vmVcenterName = null; + String vmInternalCSName = null; + for (DynamicProperty prop : props) { + if (prop.getName().equals("name")) { + vmVcenterName = prop.getVal().toString(); + } + if (prop.getName().startsWith("value[") && prop.getVal() != null) { + vmInternalCSName = ((CustomFieldStringValue)prop.getVal()).getValue(); + } + } + if (vmNameOnVcenter.equals(vmVcenterName)) { + if (vmInternalCSName != null && !vmInternalCSName.isEmpty() && !vmNameInCS.equals(vmInternalCSName)) { + return (new VirtualMachineMO(_context, oc.getObj())); + } + } + } + } + } + return null; + } + public List> getAllVmsOnDatacenter() throws Exception { List> vms = new ArrayList>(); From ad92b8507697e27ae9fd2c602cdeca61862eef7d Mon Sep 17 00:00:00 2001 From: Likitha Shetty Date: Tue, 23 Dec 2014 14:20:34 +0530 Subject: [PATCH 128/391] CLOUDSTACK-8113. VM migration fails with "Message: No such disk device: " error. Consolidate VM disks once VM/volumes are migrated. (cherry picked from commit cb211f18d14dcc9d988254a4b50b55ca0b080ed5) Signed-off-by: Rohit Yadav --- .../vmware/resource/VmwareResource.java | 24 +++++++++++++++++++ .../vmware/mo/VirtualMachineMO.java | 12 ++++++++++ 2 files changed, 36 insertions(+) diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 9ad3a5f3067..e98b82964db 100755 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -3057,6 +3057,18 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa s_logger.debug("Successfully migrated storage of VM " + vmName + " to target datastore(s)"); } + // Consolidate VM disks. + // In case of a linked clone VM, if VM's disks are not consolidated, + // further VM operations such as volume snapshot, VM snapshot etc. will result in DB inconsistencies. + String apiVersion = HypervisorHostHelper.getVcenterApiVersion(vmMo.getContext()); + if (apiVersion.compareTo("5.0") >= 0) { + if (!vmMo.consolidateVmDisks()) { + s_logger.warn("VM disk consolidation failed after storage migration. Yet proceeding with VM migration."); + } else { + s_logger.debug("Successfully consolidated disks of VM " + vmName + "."); + } + } + // Update and return volume path for every disk because that could have changed after migration for (Entry entry : volToFiler.entrySet()) { volume = entry.getKey(); @@ -3166,6 +3178,18 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa s_logger.debug("Successfully migrated volume " + volumePath + " to target datastore " + tgtDsName); } + // Consolidate VM disks. + // In case of a linked clone VM, if VM's disks are not consolidated, + // further volume operations on the ROOT volume such as volume snapshot etc. will result in DB inconsistencies. + String apiVersion = HypervisorHostHelper.getVcenterApiVersion(vmMo.getContext()); + if (apiVersion.compareTo("5.0") >= 0) { + if (!vmMo.consolidateVmDisks()) { + s_logger.warn("VM disk consolidation failed after storage migration."); + } else { + s_logger.debug("Successfully consolidated disks of VM " + vmName + "."); + } + } + // Update and return volume path because that could have changed after migration if (!targetDsMo.fileExists(fullVolumePath)) { VirtualDisk[] disks = vmMo.getAllDiskDevice(); diff --git a/vmware-base/src/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java b/vmware-base/src/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java index c2e9d7f8ee7..471b4a8ded5 100644 --- a/vmware-base/src/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java +++ b/vmware-base/src/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java @@ -2638,4 +2638,16 @@ public class VirtualMachineMO extends BaseMO { } return guestOsSupportsMemoryHotAdd && virtualHardwareSupportsMemoryHotAdd; } + + public boolean consolidateVmDisks() throws Exception { + ManagedObjectReference morTask = _context.getService().consolidateVMDisksTask(_mor); + boolean result = _context.getVimClient().waitForTask(morTask); + if (result) { + _context.waitForTaskProgressDone(morTask); + return true; + } else { + s_logger.error("VMware ConsolidateVMDisks_Task failed due to " + TaskMO.getTaskFailureInfo(_context, morTask)); + } + return false; + } } From a1eca8bd16a0548d31f69dd9b3b9d99121fa67bc Mon Sep 17 00:00:00 2001 From: Likitha Shetty Date: Fri, 28 Nov 2014 11:09:01 +0530 Subject: [PATCH 129/391] CLOUDSTACK-8118. Root volume migration fails with 'No such disk device' in case of vCenter 5.5 setup. If an exact match is being done while locating disk chain by name, don't trim snapshot postfix appended to the disk name. (cherry picked from commit ddcae8a9306f5f7dd416b859e59fd10094c91551) Signed-off-by: Rohit Yadav --- .../hypervisor/vmware/resource/VmwareResource.java | 2 +- .../cloud/hypervisor/vmware/mo/VirtualMachineMO.java | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java index e98b82964db..05313cd1f47 100755 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -3208,7 +3208,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } private int getVirtualDiskInfo(VirtualMachineMO vmMo, String srcDiskName) throws Exception { - Pair deviceInfo = vmMo.getDiskDevice(srcDiskName, true); + Pair deviceInfo = vmMo.getDiskDevice(srcDiskName, false); if (deviceInfo == null) { throw new Exception("No such disk device: " + srcDiskName); } diff --git a/vmware-base/src/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java b/vmware-base/src/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java index 471b4a8ded5..be39bfb3a5a 100644 --- a/vmware-base/src/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java +++ b/vmware-base/src/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java @@ -1931,10 +1931,13 @@ public class VirtualMachineMO extends BaseMO { DatastoreFile dsSrcFile = new DatastoreFile(vmdkDatastorePath); String srcBaseName = dsSrcFile.getFileBaseName(); + String trimmedSrcBaseName = trimSnapshotDeltaPostfix(srcBaseName); - srcBaseName = trimSnapshotDeltaPostfix(srcBaseName); - - s_logger.info("Look for disk device info from volume : " + vmdkDatastorePath + " with trimmed base name: " + srcBaseName); + if (matchExactly) { + s_logger.info("Look for disk device info from volume : " + vmdkDatastorePath + " with base name: " + srcBaseName); + } else { + s_logger.info("Look for disk device info from volume : " + vmdkDatastorePath + " with trimmed base name: " + trimmedSrcBaseName); + } if (devices != null && devices.size() > 0) { for (VirtualDevice device : devices) { @@ -1957,7 +1960,7 @@ public class VirtualMachineMO extends BaseMO { return new Pair((VirtualDisk)device, deviceNumbering); } } else { - if (backingBaseName.contains(srcBaseName)) { + if (backingBaseName.contains(trimmedSrcBaseName)) { String deviceNumbering = getDeviceBusName(devices, device); s_logger.info("Disk backing : " + diskBackingInfo.getFileName() + " matches ==> " + deviceNumbering); From 12cbebc5440de2dd1662cdde618f464c8bf3ebf7 Mon Sep 17 00:00:00 2001 From: Likitha Shetty Date: Mon, 8 Dec 2014 18:59:51 +0530 Subject: [PATCH 130/391] CLOUDSTACK-8126. Cold Migration of VM is not working as expected. In case a VM is cold migrated across clusters then VM fails to start. 1. If a VM by the same name exists on a different cluster in the VMware DC, unregister the existing VM and continue with the VM start. 2. If VM start succeeds, delete VM files associated with the unregistered VM. 3. If VM start fails, re-register the unregistered VM. (cherry picked from commit 974b0180dd67f19fea921092105161f849891ac5) Signed-off-by: Rohit Yadav --- .../vmware/resource/VmwareResource.java | 77 ++++++++++++++++++- .../vmware/mo/VirtualMachineMO.java | 36 +++++++++ 2 files changed, 111 insertions(+), 2 deletions(-) diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 05313cd1f47..89e7d3f766c 100755 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -84,6 +84,9 @@ import com.vmware.vim25.VirtualEthernetCard; import com.vmware.vim25.VirtualEthernetCardDistributedVirtualPortBackingInfo; import com.vmware.vim25.VirtualEthernetCardNetworkBackingInfo; import com.vmware.vim25.VirtualMachineConfigSpec; +import com.vmware.vim25.VirtualMachineFileInfo; +import com.vmware.vim25.VirtualMachineFileLayoutEx; +import com.vmware.vim25.VirtualMachineFileLayoutExFileInfo; import com.vmware.vim25.VirtualMachineGuestOsIdentifier; import com.vmware.vim25.VirtualMachinePowerState; import com.vmware.vim25.VirtualMachineRelocateSpec; @@ -1326,17 +1329,22 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa VirtualMachineTO vmSpec = cmd.getVirtualMachine(); boolean vmAlreadyExistsInVcenter = false; + String existingVmName = null; + VirtualMachineFileInfo existingVmFileInfo = null; + VirtualMachineFileLayoutEx existingVmFileLayout = null; + Pair names = composeVmNames(vmSpec); String vmInternalCSName = names.first(); String vmNameOnVcenter = names.second(); // Thus, vmInternalCSName always holds i-x-y, the cloudstack generated internal VM name. VmwareContext context = getServiceContext(); + DatacenterMO dcMo = null; try { VmwareManager mgr = context.getStockObject(VmwareManager.CONTEXT_STOCK_NAME); VmwareHypervisorHost hyperHost = getHyperHost(context); - DatacenterMO dcMo = new DatacenterMO(hyperHost.getContext(), hyperHost.getHyperHostDatacenter()); + dcMo = new DatacenterMO(hyperHost.getContext(), hyperHost.getHyperHostDatacenter()); // Validate VM name is unique in Datacenter VirtualMachineMO vmInVcenter = dcMo.checkIfVmAlreadyExistsInVcenter(vmNameOnVcenter, vmInternalCSName); @@ -1404,6 +1412,15 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa vmMo.tearDownDevices(new Class[] {VirtualEthernetCard.class}); vmMo.ensureScsiDeviceController(); } else { + // If a VM with the same name is found in a different cluster in the DC, unregister the old VM and configure a new VM (cold-migration). + VirtualMachineMO existingVmInDc = dcMo.findVm(vmInternalCSName); + if (existingVmInDc != null) { + s_logger.debug("Found VM: " + vmInternalCSName + " on a host in a different cluster. Unregistering the exisitng VM."); + existingVmName = existingVmInDc.getName(); + existingVmFileInfo = existingVmInDc.getFileInfo(); + existingVmFileLayout = existingVmInDc.getFileLayout(); + existingVmInDc.unregisterVm(); + } Pair rootDiskDataStoreDetails = null; for (DiskTO vol : disks) { if (vol.getType() == Volume.Type.ROOT) { @@ -1429,7 +1446,9 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa assert (vmSpec.getMinSpeed() != null) && (rootDiskDataStoreDetails != null); - if (rootDiskDataStoreDetails.second().folderExists(String.format("[%s]", rootDiskDataStoreDetails.second().getName()), vmNameOnVcenter)) { + boolean vmFolderExists = rootDiskDataStoreDetails.second().folderExists(String.format("[%s]", rootDiskDataStoreDetails.second().getName()), vmNameOnVcenter); + String vmxFileFullPath = dsRootVolumeIsOn.searchFileInSubFolders(vmNameOnVcenter + ".vmx", false); + if (vmFolderExists && vmxFileFullPath != null) { // VM can be registered only if .vmx is present. registerVm(vmNameOnVcenter, dsRootVolumeIsOn); vmMo = hyperHost.findVmOnHyperHost(vmInternalCSName); tearDownVm(vmMo); @@ -1740,6 +1759,11 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa startAnswer.setIqnToPath(iqnToPath); + // Since VM was successfully powered-on, if there was an existing VM in a different cluster that was unregistered, delete all the files associated with it. + if (existingVmName != null && existingVmFileLayout != null) { + deleteUnregisteredVmFiles(existingVmFileLayout, dcMo); + } + return startAnswer; } catch (Throwable e) { if (e instanceof RemoteException) { @@ -1753,6 +1777,20 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa if(vmAlreadyExistsInVcenter) { startAnswer.setContextParam("stopRetry", "true"); } + + // Since VM start failed, if there was an existing VM in a different cluster that was unregistered, register it back. + if (existingVmName != null && existingVmFileInfo != null) { + s_logger.debug("Since VM start failed, registering back an existing VM: " + existingVmName + " that was unregistered"); + try { + DatastoreFile fileInDatastore = new DatastoreFile(existingVmFileInfo.getVmPathName()); + DatastoreMO existingVmDsMo = new DatastoreMO(dcMo.getContext(), dcMo.findDatastore(fileInDatastore.getDatastoreName())); + registerVm(existingVmName, existingVmDsMo); + } catch (Exception ex){ + String message = "Failed to register an existing VM: " + existingVmName + " due to " + VmwareHelper.getExceptionMessage(ex); + s_logger.warn(message, ex); + } + } + return startAnswer; } finally { } @@ -2204,6 +2242,41 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } } + private void deleteUnregisteredVmFiles(VirtualMachineFileLayoutEx vmFileLayout, DatacenterMO dcMo) throws Exception { + s_logger.debug("Deleting files associated with an existing VM that was unregistered"); + DatastoreFile vmFolder = null; + try { + List fileInfo = vmFileLayout.getFile(); + for (VirtualMachineFileLayoutExFileInfo file : fileInfo) { + DatastoreFile fileInDatastore = new DatastoreFile(file.getName()); + // In case of linked clones, VM file layout includes the base disk so don't delete all disk files. + if (file.getType().startsWith("disk") || file.getType().startsWith("digest")) + continue; + else if (file.getType().equals("config")) + vmFolder = new DatastoreFile(fileInDatastore.getDatastoreName(), fileInDatastore.getDir()); + DatastoreMO dsMo = new DatastoreMO(dcMo.getContext(), dcMo.findDatastore(fileInDatastore.getDatastoreName())); + s_logger.debug("Deleting file: " + file.getName()); + dsMo.deleteFile(file.getName(), dcMo.getMor(), true); + } + // Delete files that are present in the VM folder - this will take care of the VM disks as well. + DatastoreMO vmFolderDsMo = new DatastoreMO(dcMo.getContext(), dcMo.findDatastore(vmFolder.getDatastoreName())); + String[] files = vmFolderDsMo.listDirContent(vmFolder.getPath()); + if (files.length != 0) { + for (String file : files) { + String vmDiskFileFullPath = String.format("%s/%s", vmFolder.getPath(), file); + s_logger.debug("Deleting file: " + vmDiskFileFullPath); + vmFolderDsMo.deleteFile(vmDiskFileFullPath, dcMo.getMor(), true); + } + } + // Delete VM folder + s_logger.debug("Deleting folder: " + vmFolder.getPath()); + vmFolderDsMo.deleteFolder(vmFolder.getPath(), dcMo.getMor()); + } catch (Exception e) { + String message = "Failed to delete files associated with an existing VM that was unregistered due to " + VmwareHelper.getExceptionMessage(e); + s_logger.warn(message, e); + } + } + private static VolumeObjectTO getVolumeInSpec(VirtualMachineTO vmSpec, VolumeObjectTO srcVol) { for (DiskTO disk : vmSpec.getDisks()) { VolumeObjectTO vol = (VolumeObjectTO)disk.getData(); diff --git a/vmware-base/src/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java b/vmware-base/src/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java index be39bfb3a5a..4517ad4e637 100644 --- a/vmware-base/src/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java +++ b/vmware-base/src/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java @@ -87,6 +87,7 @@ import com.vmware.vim25.VirtualMachineConfigOption; import com.vmware.vim25.VirtualMachineConfigSpec; import com.vmware.vim25.VirtualMachineConfigSummary; import com.vmware.vim25.VirtualMachineFileInfo; +import com.vmware.vim25.VirtualMachineFileLayoutEx; import com.vmware.vim25.VirtualMachineMessage; import com.vmware.vim25.VirtualMachineMovePriority; import com.vmware.vim25.VirtualMachinePowerState; @@ -741,6 +742,41 @@ public class VirtualMachineMO extends BaseMO { return (VirtualMachineFileInfo)_context.getVimClient().getDynamicProperty(_mor, "config.files"); } + public VirtualMachineFileLayoutEx getFileLayout() throws Exception { + VirtualMachineFileLayoutEx fileLayout = null; + PropertySpec pSpec = new PropertySpec(); + pSpec.setType("VirtualMachine"); + pSpec.getPathSet().add("layoutEx"); + + ObjectSpec oSpec = new ObjectSpec(); + oSpec.setObj(_mor); + oSpec.setSkip(Boolean.FALSE); + + PropertyFilterSpec pfSpec = new PropertyFilterSpec(); + pfSpec.getPropSet().add(pSpec); + pfSpec.getObjectSet().add(oSpec); + List pfSpecArr = new ArrayList(); + pfSpecArr.add(pfSpec); + + List ocs = _context.getService().retrieveProperties(_context.getPropertyCollector(), pfSpecArr); + + if (ocs != null) { + for (ObjectContent oc : ocs) { + List props = oc.getPropSet(); + if (props != null) { + for (DynamicProperty prop : props) { + if (prop.getName().equals("layoutEx")) { + fileLayout = (VirtualMachineFileLayoutEx)prop.getVal(); + break; + } + } + } + } + } + + return fileLayout; + } + @Override public ManagedObjectReference getParentMor() throws Exception { return (ManagedObjectReference)_context.getVimClient().getDynamicProperty(_mor, "parent"); From bcbfe3bdeefedf4f53e1b5bade4e220fb8d23a57 Mon Sep 17 00:00:00 2001 From: Likitha Shetty Date: Thu, 18 Dec 2014 19:38:14 +0530 Subject: [PATCH 131/391] CLOUDSTACK-8129. Cold migration of VM across VMware DCs leaves the VM behind in the source host. If VM has been cold migrated across different VMware DCs, then unregister the VM from source host. (cherry picked from commit 15b348632df2049347f58c87830be2c02eee3b61) Signed-off-by: Rohit Yadav --- .../cloud/agent/api/UnregisterVMCommand.java | 9 +++++++ .../cloud/vm/VirtualMachineManagerImpl.java | 27 +++++++++++++++++++ .../src/com/cloud/dc/ClusterDetailsDao.java | 2 ++ .../com/cloud/dc/ClusterDetailsDaoImpl.java | 10 +++++++ .../vmware/resource/VmwareResource.java | 18 +++++++++---- .../cloud/storage/VolumeApiServiceImpl.java | 19 +++++++++++++ 6 files changed, 80 insertions(+), 5 deletions(-) diff --git a/core/src/com/cloud/agent/api/UnregisterVMCommand.java b/core/src/com/cloud/agent/api/UnregisterVMCommand.java index a0085e0ca02..16eb4bac0e8 100644 --- a/core/src/com/cloud/agent/api/UnregisterVMCommand.java +++ b/core/src/com/cloud/agent/api/UnregisterVMCommand.java @@ -21,6 +21,7 @@ package com.cloud.agent.api; public class UnregisterVMCommand extends Command { String vmName; + boolean cleanupVmFiles = false; public UnregisterVMCommand(String vmName) { this.vmName = vmName; @@ -34,4 +35,12 @@ public class UnregisterVMCommand extends Command { public String getVmName() { return vmName; } + + public void setCleanupVmFiles(boolean cleanupVmFiles) { + this.cleanupVmFiles = cleanupVmFiles; + } + + public boolean getCleanupVmFiles() { + return this.cleanupVmFiles; + } } diff --git a/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java index 8c7388d342a..eed0963449e 100755 --- a/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java @@ -95,6 +95,7 @@ import com.cloud.agent.api.StopAnswer; import com.cloud.agent.api.StopCommand; import com.cloud.agent.api.UnPlugNicAnswer; import com.cloud.agent.api.UnPlugNicCommand; +import com.cloud.agent.api.UnregisterVMCommand; import com.cloud.agent.api.to.DiskTO; import com.cloud.agent.api.to.GPUDeviceTO; import com.cloud.agent.api.to.NicTO; @@ -1716,6 +1717,9 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac private void orchestrateStorageMigration(String vmUuid, StoragePool destPool) { VMInstanceVO vm = _vmDao.findByUuid(vmUuid); + Long srchostId = vm.getHostId() != null ? vm.getHostId() : vm.getLastHostId(); + HostVO srcHost = _hostDao.findById(srchostId); + Long srcClusterId = srcHost.getClusterId(); try { stateTransitTo(vm, VirtualMachine.Event.StorageMigrationRequested, null); @@ -1741,6 +1745,29 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac //when start the vm next time, don;'t look at last_host_id, only choose the host based on volume/storage pool vm.setLastHostId(null); vm.setPodIdToDeployIn(destPool.getPodId()); + + // If VM was cold migrated between clusters belonging to two different VMware DCs, + // unregister the VM from the source host and cleanup the associated VM files. + if (vm.getHypervisorType().equals(HypervisorType.VMware)) { + Long destClusterId = destPool.getClusterId(); + if (srcClusterId != null && destClusterId != null && srcClusterId != destClusterId) { + String srcDcName = _clusterDetailsDao.getVmwareDcName(srcClusterId); + String destDcName = _clusterDetailsDao.getVmwareDcName(destClusterId); + if (srcDcName != null && destDcName != null && !srcDcName.equals(destDcName)) { + s_logger.debug("Since VM's storage was successfully migrated across VMware Datacenters, unregistering VM: " + vm.getInstanceName() + + " from source host: " + srcHost.getId()); + UnregisterVMCommand uvc = new UnregisterVMCommand(vm.getInstanceName()); + uvc.setCleanupVmFiles(true); + try { + _agentMgr.send(srcHost.getId(), uvc); + } catch (Exception e) { + throw new CloudRuntimeException("Failed to unregister VM: " + vm.getInstanceName() + " from source host: " + srcHost.getId() + + " after successfully migrating VM's storage across VMware Datacenters"); + } + } + } + } + } else { s_logger.debug("Storage migration failed"); } diff --git a/engine/schema/src/com/cloud/dc/ClusterDetailsDao.java b/engine/schema/src/com/cloud/dc/ClusterDetailsDao.java index 49250d94be6..06c9c525504 100644 --- a/engine/schema/src/com/cloud/dc/ClusterDetailsDao.java +++ b/engine/schema/src/com/cloud/dc/ClusterDetailsDao.java @@ -30,4 +30,6 @@ public interface ClusterDetailsDao extends GenericDao { ClusterDetailsVO findDetail(long clusterId, String name); void deleteDetails(long clusterId); + + String getVmwareDcName(Long clusterId); } diff --git a/engine/schema/src/com/cloud/dc/ClusterDetailsDaoImpl.java b/engine/schema/src/com/cloud/dc/ClusterDetailsDaoImpl.java index 0d6b833b399..c9397c2574f 100755 --- a/engine/schema/src/com/cloud/dc/ClusterDetailsDaoImpl.java +++ b/engine/schema/src/com/cloud/dc/ClusterDetailsDaoImpl.java @@ -139,4 +139,14 @@ public class ClusterDetailsDaoImpl extends GenericDaoBase 3) + dcName = tokens[3]; + return dcName; + } } diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 89e7d3f766c..315b1619f4a 100755 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -1761,7 +1761,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa // Since VM was successfully powered-on, if there was an existing VM in a different cluster that was unregistered, delete all the files associated with it. if (existingVmName != null && existingVmFileLayout != null) { - deleteUnregisteredVmFiles(existingVmFileLayout, dcMo); + deleteUnregisteredVmFiles(existingVmFileLayout, dcMo, true); } return startAnswer; @@ -2242,7 +2242,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } } - private void deleteUnregisteredVmFiles(VirtualMachineFileLayoutEx vmFileLayout, DatacenterMO dcMo) throws Exception { + private void deleteUnregisteredVmFiles(VirtualMachineFileLayoutEx vmFileLayout, DatacenterMO dcMo, boolean deleteDisks) throws Exception { s_logger.debug("Deleting files associated with an existing VM that was unregistered"); DatastoreFile vmFolder = null; try { @@ -2261,7 +2261,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa // Delete files that are present in the VM folder - this will take care of the VM disks as well. DatastoreMO vmFolderDsMo = new DatastoreMO(dcMo.getContext(), dcMo.findDatastore(vmFolder.getDatastoreName())); String[] files = vmFolderDsMo.listDirContent(vmFolder.getPath()); - if (files.length != 0) { + if (deleteDisks) { for (String file : files) { String vmDiskFileFullPath = String.format("%s/%s", vmFolder.getPath(), file); s_logger.debug("Deleting file: " + vmDiskFileFullPath); @@ -2269,8 +2269,10 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa } } // Delete VM folder - s_logger.debug("Deleting folder: " + vmFolder.getPath()); - vmFolderDsMo.deleteFolder(vmFolder.getPath(), dcMo.getMor()); + if (deleteDisks || files.length == 0) { + s_logger.debug("Deleting folder: " + vmFolder.getPath()); + vmFolderDsMo.deleteFolder(vmFolder.getPath(), dcMo.getMor()); + } } catch (Exception e) { String message = "Failed to delete files associated with an existing VM that was unregistered due to " + VmwareHelper.getExceptionMessage(e); s_logger.warn(message, e); @@ -2768,6 +2770,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa try { vmMo.setCustomFieldValue(CustomFieldConstants.CLOUD_NIC_MASK, "0"); + vmMo.setCustomFieldValue(CustomFieldConstants.CLOUD_VM_INTERNAL_NAME, cmd.getVmName()); if (getVmPowerState(vmMo) != PowerState.PowerOff) { if (vmMo.safePowerOff(_shutdownWaitMs)) { @@ -3898,10 +3901,15 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa VmwareContext context = getServiceContext(); VmwareHypervisorHost hyperHost = getHyperHost(context); try { + DatacenterMO dataCenterMo = new DatacenterMO(getServiceContext(), hyperHost.getHyperHostDatacenter()); VirtualMachineMO vmMo = hyperHost.findVmOnHyperHost(cmd.getVmName()); if (vmMo != null) { try { + VirtualMachineFileLayoutEx vmFileLayout = vmMo.getFileLayout(); context.getService().unregisterVM(vmMo.getMor()); + if (cmd.getCleanupVmFiles()) { + deleteUnregisteredVmFiles(vmFileLayout, dataCenterMo, false); + } return new Answer(cmd, true, "unregister succeeded"); } catch (Exception e) { s_logger.warn("We are not able to unregister VM " + VmwareHelper.getExceptionMessage(e)); diff --git a/server/src/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/com/cloud/storage/VolumeApiServiceImpl.java index 40b957446af..71042c50dba 100644 --- a/server/src/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/com/cloud/storage/VolumeApiServiceImpl.java @@ -79,6 +79,7 @@ import com.cloud.api.ApiDBUtils; import com.cloud.configuration.Config; import com.cloud.configuration.ConfigurationManager; import com.cloud.configuration.Resource.ResourceType; +import com.cloud.dc.ClusterDetailsDao; import com.cloud.dc.ClusterVO; import com.cloud.dc.DataCenter; import com.cloud.dc.DataCenterVO; @@ -221,6 +222,8 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic AsyncJobManager _jobMgr; @Inject VmWorkJobDao _workJobDao; + @Inject + ClusterDetailsDao _clusterDetailsDao; private List _storagePoolAllocators; @@ -1764,6 +1767,22 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic throw new InvalidParameterValueException("Cannot migrate a volume of a virtual machine to a storage pool in a different cluster"); } } + // In case of VMware, if ROOT volume is being cold-migrated, then ensure destination storage pool is in the same Datacenter as the VM. + if (vm != null && vm.getHypervisorType().equals(HypervisorType.VMware)) { + if (!liveMigrateVolume && vol.volumeType.equals(Volume.Type.ROOT)) { + Long hostId = vm.getHostId() != null ? vm.getHostId() : vm.getLastHostId(); + HostVO host = _hostDao.findById(hostId); + if (host != null) + srcClusterId = host.getClusterId(); + if (srcClusterId != null && destPool.getClusterId() != null && !srcClusterId.equals(destPool.getClusterId())) { + String srcDcName = _clusterDetailsDao.getVmwareDcName(srcClusterId); + String destDcName = _clusterDetailsDao.getVmwareDcName(destPool.getClusterId()); + if (srcDcName != null && destDcName != null && !srcDcName.equals(destDcName)) { + throw new InvalidParameterValueException("Cannot migrate ROOT volume of a stopped VM to a storage pool in a different VMware datacenter"); + } + } + } + } } } else { throw new InvalidParameterValueException("Migration of volume from local storage pool is not supported"); From b9932a0129c565db185e8f5ee03d68dbc7bce107 Mon Sep 17 00:00:00 2001 From: Likitha Shetty Date: Fri, 5 Dec 2014 16:00:21 +0530 Subject: [PATCH 132/391] CLOUDSTACK-8122. Handle NPE thrown during migration failures. When migration fails instead of returning NULL, throw the exception. (cherry picked from commit a5a65c7b551ee5cc32588997937267b716eff681) Signed-off-by: Rohit Yadav --- .../orchestration/VolumeOrchestrator.java | 4 +-- .../cloud/storage/VolumeApiServiceImpl.java | 26 ++++++++++++------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java index 1a3c0b6dd86..e439b33b2bf 100644 --- a/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java +++ b/engine/orchestration/src/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java @@ -934,10 +934,10 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati return result.getVolume(); } catch (InterruptedException e) { s_logger.debug("migrate volume failed", e); - return null; + throw new CloudRuntimeException(e.getMessage()); } catch (ExecutionException e) { s_logger.debug("migrate volume failed", e); - return null; + throw new CloudRuntimeException(e.getMessage()); } } diff --git a/server/src/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/com/cloud/storage/VolumeApiServiceImpl.java index 71042c50dba..6d74fa43eb7 100644 --- a/server/src/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/com/cloud/storage/VolumeApiServiceImpl.java @@ -1817,6 +1817,8 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic if (jobResult != null) { if (jobResult instanceof ConcurrentOperationException) throw (ConcurrentOperationException)jobResult; + else if (jobResult instanceof RuntimeException) + throw (RuntimeException)jobResult; else if (jobResult instanceof Throwable) throw new RuntimeException("Unexpected exception", (Throwable)jobResult); } @@ -1839,35 +1841,39 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic assert (destPool != null); Volume newVol = null; - if (liveMigrateVolume) { - newVol = liveMigrateVolume(vol, destPool); - } else { - try { + try { + if (liveMigrateVolume) { + newVol = liveMigrateVolume(vol, destPool); + } else { newVol = _volumeMgr.migrateVolume(vol, destPool); - } catch (StorageUnavailableException e) { - s_logger.debug("Failed to migrate volume", e); } + } catch (StorageUnavailableException e) { + s_logger.debug("Failed to migrate volume", e); + throw new CloudRuntimeException(e.getMessage()); + } catch (Exception e) { + s_logger.debug("Failed to migrate volume", e); + throw new CloudRuntimeException(e.getMessage()); } return newVol; } @DB - protected Volume liveMigrateVolume(Volume volume, StoragePool destPool) { + protected Volume liveMigrateVolume(Volume volume, StoragePool destPool) throws StorageUnavailableException { VolumeInfo vol = volFactory.getVolume(volume.getId()); AsyncCallFuture future = volService.migrateVolume(vol, (DataStore)destPool); try { VolumeApiResult result = future.get(); if (result.isFailed()) { s_logger.debug("migrate volume failed:" + result.getResult()); - return null; + throw new StorageUnavailableException("Migrate volume failed: " + result.getResult(), destPool.getId()); } return result.getVolume(); } catch (InterruptedException e) { s_logger.debug("migrate volume failed", e); - return null; + throw new CloudRuntimeException(e.getMessage()); } catch (ExecutionException e) { s_logger.debug("migrate volume failed", e); - return null; + throw new CloudRuntimeException(e.getMessage()); } } From 272f282a056c532adefc3fef6076e6337a6a7179 Mon Sep 17 00:00:00 2001 From: Likitha Shetty Date: Tue, 2 Dec 2014 16:50:20 +0530 Subject: [PATCH 133/391] CLOUDSTACK-8119. Propagate error message to UI for attach/detach volume failure operations. For AttachVolume/DetachVolume API command, improve user error message in case of RuntimeException by throwing the exception instead of 'Unexpected Exception'. (cherry picked from commit 4d7ede535df568c6aab4a228ac794ec11d433e1e) Signed-off-by: Rohit Yadav Conflicts: server/src/com/cloud/storage/VolumeApiServiceImpl.java --- .../cloud/storage/resource/VmwareStorageProcessor.java | 6 +++++- server/src/com/cloud/storage/VolumeApiServiceImpl.java | 6 ++++++ .../com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java | 8 ++++++++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageProcessor.java index 621f005bd57..620d5fae979 100644 --- a/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -1357,7 +1357,11 @@ public class VmwareStorageProcessor implements StorageProcessor { hostService.invalidateServiceContext(null); } - String msg = "AttachVolumeCommand failed due to " + VmwareHelper.getExceptionMessage(e); + String msg = ""; + if (isAttach) + msg += "Failed to attach volume: " + e.getMessage(); + else + msg += "Failed to detach volume: " + e.getMessage(); s_logger.error(msg, e); return new AttachAnswer(msg); } diff --git a/server/src/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/com/cloud/storage/VolumeApiServiceImpl.java index 6d74fa43eb7..fe46b1cb503 100644 --- a/server/src/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/com/cloud/storage/VolumeApiServiceImpl.java @@ -1387,6 +1387,10 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic if (jobResult != null) { if (jobResult instanceof ConcurrentOperationException) throw (ConcurrentOperationException)jobResult; + else if (jobResult instanceof InvalidParameterValueException) + throw (InvalidParameterValueException)jobResult; + else if (jobResult instanceof RuntimeException) + throw (RuntimeException)jobResult; else if (jobResult instanceof Throwable) throw new RuntimeException("Unexpected exception", (Throwable)jobResult); else if (jobResult instanceof Long) { @@ -1590,6 +1594,8 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic if (jobResult != null) { if (jobResult instanceof ConcurrentOperationException) throw (ConcurrentOperationException)jobResult; + else if (jobResult instanceof RuntimeException) + throw (RuntimeException)jobResult; else if (jobResult instanceof Throwable) throw new RuntimeException("Unexpected exception", (Throwable)jobResult); else if (jobResult instanceof Long) { diff --git a/vmware-base/src/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java b/vmware-base/src/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java index 4517ad4e637..f8924779014 100644 --- a/vmware-base/src/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java +++ b/vmware-base/src/com/cloud/hypervisor/vmware/mo/VirtualMachineMO.java @@ -1164,6 +1164,14 @@ public class VirtualMachineMO extends BaseMO { throw new Exception("No such disk device: " + vmdkDatastorePath); } + // IDE virtual disk cannot be detached if VM is running + if (deviceInfo.second() != null && deviceInfo.second().contains("ide")) { + if (getPowerState() == VirtualMachinePowerState.POWERED_ON) { + throw new Exception("Removing a virtual disk over IDE controller is not supported while VM is running in VMware hypervisor. " + + "Please re-try when VM is not running."); + } + } + List> chain = getDiskDatastorePathChain(deviceInfo.first(), true); VirtualMachineConfigSpec reConfigSpec = new VirtualMachineConfigSpec(); From 294f5bf331429c61b10cd426c111e39dd7619b9d Mon Sep 17 00:00:00 2001 From: Likitha Shetty Date: Wed, 5 Nov 2014 14:51:10 +0530 Subject: [PATCH 134/391] CLOUDSTACK-8114. Ensure VM stop and then start updates the volume path correctly in the DB. (cherry picked from commit 521258bafb91e30d7bd43fba62cc47a77c078028) Signed-off-by: Rohit Yadav --- .../src/com/cloud/vm/VirtualMachineManagerImpl.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java index eed0963449e..8b8c83b14a7 100755 --- a/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java @@ -1201,7 +1201,11 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac // Use getPath() from VolumeVO to get a fresh copy of what's in the DB. // Before doing this, in a certain situation, getPath() from VolumeObjectTO // returned null instead of an actual path (because it was out of date with the DB). - volumeMgr.updateVolumeDiskChain(vol.getId(), volume.getPath(), vol.getChainInfo()); + if(vol.getPath() != null) { + volumeMgr.updateVolumeDiskChain(vol.getId(), vol.getPath(), vol.getChainInfo()); + } else { + volumeMgr.updateVolumeDiskChain(vol.getId(), volume.getPath(), vol.getChainInfo()); + } } } } From 0c1172ffe93b6283b5e849898267d2aa8b2128b3 Mon Sep 17 00:00:00 2001 From: Kishan Kavala Date: Fri, 21 Nov 2014 12:10:50 +0530 Subject: [PATCH 135/391] Network offering usage event should be logged for UserVms only (cherry picked from commit 42cecbb000416978a7edcef5d8c664dac4b52277) Signed-off-by: Rohit Yadav --- .../src/com/cloud/vm/VirtualMachineManagerImpl.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java index 8b8c83b14a7..4c90dbe2721 100755 --- a/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/com/cloud/vm/VirtualMachineManagerImpl.java @@ -2924,8 +2924,11 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac s_logger.debug("Nic is plugged successfully for vm " + vm + " in network " + network + ". Vm is a part of network now"); long isDefault = (nic.isDefaultNic()) ? 1 : 0; // insert nic's Id into DB as resource_name - UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NETWORK_OFFERING_ASSIGN, vmVO.getAccountId(), vmVO.getDataCenterId(), vmVO.getId(), - Long.toString(nic.getId()), network.getNetworkOfferingId(), null, isDefault, VirtualMachine.class.getName(), vmVO.getUuid(), vm.isDisplay()); + if(VirtualMachine.Type.User.equals(vmVO.getType())) { + //Log usage event for user Vms only + UsageEventUtils.publishUsageEvent(EventTypes.EVENT_NETWORK_OFFERING_ASSIGN, vmVO.getAccountId(), vmVO.getDataCenterId(), vmVO.getId(), + Long.toString(nic.getId()), network.getNetworkOfferingId(), null, isDefault, VirtualMachine.class.getName(), vmVO.getUuid(), vm.isDisplay()); + } return nic; } else { s_logger.warn("Failed to plug nic to the vm " + vm + " in network " + network); From f00140243c8628757a1474310af192eb932aab6e Mon Sep 17 00:00:00 2001 From: Likitha Shetty Date: Wed, 12 Nov 2014 17:27:21 +0530 Subject: [PATCH 136/391] CLOUDSTACK-8112. CS allows creation of VM's with the same Display name when vm.instancename.flag is set to true. During VM creation, if vm.instancename.flag is set to true and hypervisor type is VMware, check if VM with the same hostname already exists in the zone. (cherry picked from commit 5f9e4fddf303f312a0b17abc0d837f28042caeda) Signed-off-by: Rohit Yadav --- .../schema/src/com/cloud/vm/dao/VMInstanceDao.java | 2 ++ .../src/com/cloud/vm/dao/VMInstanceDaoImpl.java | 14 ++++++++++++++ server/src/com/cloud/vm/UserVmManagerImpl.java | 6 ++++++ 3 files changed, 22 insertions(+) diff --git a/engine/schema/src/com/cloud/vm/dao/VMInstanceDao.java b/engine/schema/src/com/cloud/vm/dao/VMInstanceDao.java index 6ba7c365215..1e4c8b66c0e 100644 --- a/engine/schema/src/com/cloud/vm/dao/VMInstanceDao.java +++ b/engine/schema/src/com/cloud/vm/dao/VMInstanceDao.java @@ -136,4 +136,6 @@ public interface VMInstanceDao extends GenericDao, StateDao< void resetHostPowerStateTracking(long hostId); HashMap countVgpuVMs(Long dcId, Long podId, Long clusterId); + + VMInstanceVO findVMByHostNameInZone(String hostName, long zoneId); } diff --git a/engine/schema/src/com/cloud/vm/dao/VMInstanceDaoImpl.java b/engine/schema/src/com/cloud/vm/dao/VMInstanceDaoImpl.java index df023bfcf79..3eabbdb7499 100644 --- a/engine/schema/src/com/cloud/vm/dao/VMInstanceDaoImpl.java +++ b/engine/schema/src/com/cloud/vm/dao/VMInstanceDaoImpl.java @@ -85,6 +85,7 @@ public class VMInstanceDaoImpl extends GenericDaoBase implem protected SearchBuilder HostUpSearch; protected SearchBuilder InstanceNameSearch; protected SearchBuilder HostNameSearch; + protected SearchBuilder HostNameAndZoneSearch; protected GenericSearchBuilder FindIdsOfVirtualRoutersByAccount; protected GenericSearchBuilder CountRunningByHost; protected GenericSearchBuilder CountRunningByAccount; @@ -218,6 +219,11 @@ public class VMInstanceDaoImpl extends GenericDaoBase implem HostNameSearch.and("hostName", HostNameSearch.entity().getHostName(), Op.EQ); HostNameSearch.done(); + HostNameAndZoneSearch = createSearchBuilder(); + HostNameAndZoneSearch.and("hostName", HostNameAndZoneSearch.entity().getHostName(), Op.EQ); + HostNameAndZoneSearch.and("zone", HostNameAndZoneSearch.entity().getDataCenterId(), Op.EQ); + HostNameAndZoneSearch.done(); + FindIdsOfVirtualRoutersByAccount = createSearchBuilder(Long.class); FindIdsOfVirtualRoutersByAccount.selectFields(FindIdsOfVirtualRoutersByAccount.entity().getId()); FindIdsOfVirtualRoutersByAccount.and("account", FindIdsOfVirtualRoutersByAccount.entity().getAccountId(), SearchCriteria.Op.EQ); @@ -414,6 +420,14 @@ public class VMInstanceDaoImpl extends GenericDaoBase implem return findOneBy(sc); } + @Override + public VMInstanceVO findVMByHostNameInZone(String hostName, long zoneId) { + SearchCriteria sc = HostNameAndZoneSearch.create(); + sc.setParameters("hostName", hostName); + sc.setParameters("zone", zoneId); + return findOneBy(sc); + } + @Override public void updateProxyId(long id, Long proxyId, Date time) { VMInstanceVO vo = createForUpdate(); diff --git a/server/src/com/cloud/vm/UserVmManagerImpl.java b/server/src/com/cloud/vm/UserVmManagerImpl.java index 22f27691c07..c74c74efc7e 100755 --- a/server/src/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/com/cloud/vm/UserVmManagerImpl.java @@ -2919,6 +2919,12 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir hostName = generateHostName(uuidName); } } + // If global config vm.instancename.flag is set to true, then CS will set guest VM's name as it appears on the hypervisor, to its hostname. + // In case of VMware since VM name must be unique within a DC, check if VM with the same hostname already exists in the zone. + VMInstanceVO vmByHostName = _vmInstanceDao.findVMByHostNameInZone(hostName, zone.getId()); + if (vmByHostName != null && vmByHostName.getState() != VirtualMachine.State.Expunging) { + throw new InvalidParameterValueException("There already exists a VM by the name: " + hostName + "."); + } } else { if (hostName == null) { //Generate name using uuid and instance.name global config From 0e4d91aa91c51b89466293754622b7a4289166f9 Mon Sep 17 00:00:00 2001 From: Devdeep Singh Date: Fri, 14 Nov 2014 11:11:49 +0530 Subject: [PATCH 137/391] CLOUDSTACK-6924. To attach a volume if a volume needs to be moved to another storage pool, the source and destination pools cannot be local and cluster/zone and vice versa. Cloudstack detects it and throws a exception. However, the end user only sees an unexpected exception and not the reason for failure. Fixed it by making sure the reason for the failure is correctly captured and shown to the end user. (cherry picked from commit cffae8eef0b1f9bf869a5ec99befbe9ae9d9290d) Signed-off-by: Rohit Yadav Conflicts: server/src/com/cloud/storage/VolumeApiServiceImpl.java --- .../orchestration/src/com/cloud/vm/VmWorkJobDispatcher.java | 4 ++++ server/src/com/cloud/storage/VolumeApiServiceImpl.java | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/engine/orchestration/src/com/cloud/vm/VmWorkJobDispatcher.java b/engine/orchestration/src/com/cloud/vm/VmWorkJobDispatcher.java index c023511a224..bbbf7fd4026 100644 --- a/engine/orchestration/src/com/cloud/vm/VmWorkJobDispatcher.java +++ b/engine/orchestration/src/com/cloud/vm/VmWorkJobDispatcher.java @@ -28,6 +28,7 @@ import org.apache.cloudstack.framework.jobs.AsyncJobDispatcher; import org.apache.cloudstack.framework.jobs.AsyncJobManager; import org.apache.cloudstack.jobs.JobInfo; +import com.cloud.exception.InvalidParameterValueException; import com.cloud.utils.Pair; import com.cloud.utils.component.AdapterBase; import com.cloud.vm.dao.VMInstanceDao; @@ -108,6 +109,9 @@ public class VmWorkJobDispatcher extends AdapterBase implements AsyncJobDispatch if (s_logger.isDebugEnabled()) s_logger.debug("Done with run of VM work job: " + cmd + " for VM " + work.getVmId() + ", job origin: " + job.getRelated()); } + } catch(InvalidParameterValueException e) { + s_logger.error("Unable to complete " + job + ", job origin:" + job.getRelated()); + _asyncJobMgr.completeAsyncJob(job.getId(), JobInfo.Status.FAILED, 0, _asyncJobMgr.marshallResultObject(e)); } catch(Throwable e) { s_logger.error("Unable to complete " + job + ", job origin:" + job.getRelated(), e); diff --git a/server/src/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/com/cloud/storage/VolumeApiServiceImpl.java index fe46b1cb503..219a48299c7 100644 --- a/server/src/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/com/cloud/storage/VolumeApiServiceImpl.java @@ -2244,7 +2244,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic return false; } } - throw new CloudRuntimeException("Can't move volume between scope: " + storeForNewStoreScope.getScopeType() + " and " + storeForExistingStoreScope.getScopeType()); + throw new InvalidParameterValueException("Can't move volume between scope: " + storeForNewStoreScope.getScopeType() + " and " + storeForExistingStoreScope.getScopeType()); } return !storeForExistingStoreScope.isSameScope(storeForNewStoreScope); From d466ecaf7a05a6c5ecdc8ef61aff34c930bc12d4 Mon Sep 17 00:00:00 2001 From: Likitha Shetty Date: Tue, 6 Jan 2015 15:33:47 +0530 Subject: [PATCH 138/391] CLOUDSTACK-5485. Cannot process more than 5 concurrent snapshots. Number of threads on SSVM agent for connection with MS (Agent->NioClient) should be configurable using global config 'workers'. (cherry picked from commit bc235ed5eb0f88c3c62542a3e06bdc28c7b49eac) Signed-off-by: Rohit Yadav --- agent/src/com/cloud/agent/AgentShell.java | 3 +++ .../secondarystorage/SecondaryStorageManagerImpl.java | 2 ++ 2 files changed, 5 insertions(+) diff --git a/agent/src/com/cloud/agent/AgentShell.java b/agent/src/com/cloud/agent/AgentShell.java index 67b7b2fa4ad..78430aee0b5 100644 --- a/agent/src/com/cloud/agent/AgentShell.java +++ b/agent/src/com/cloud/agent/AgentShell.java @@ -236,6 +236,9 @@ public class AgentShell implements IAgentShell, Daemon { } _workers = NumberUtils.toInt(workers, 5); + if (_workers <= 0) { + _workers = 5; + } if (host == null) { host = getProperty(null, "host"); diff --git a/services/secondary-storage/controller/src/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java b/services/secondary-storage/controller/src/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java index cc7eb253bb0..e8857f16a00 100755 --- a/services/secondary-storage/controller/src/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java +++ b/services/secondary-storage/controller/src/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java @@ -1051,6 +1051,8 @@ public class SecondaryStorageManagerImpl extends ManagerBase implements Secondar buf.append(" guid=").append(profile.getVirtualMachine().getHostName()); + buf.append(" workers=").append(_configDao.getValue("workers")); + if (_configDao.isPremium()) { s_logger.debug("VmWare hypervisor configured, telling the ssvm to load the PremiumSecondaryStorageResource"); buf.append(" resource=com.cloud.storage.resource.PremiumSecondaryStorageResource"); From bf7013f0ea4ce1102484d64bb8439d193c0cee84 Mon Sep 17 00:00:00 2001 From: Likitha Shetty Date: Thu, 20 Mar 2014 11:29:57 +0530 Subject: [PATCH 139/391] CLOUDSTACK-8111. NFS secondary storage repetitively mounted on CS server with ESXi hypervisors. Fix cleanup of NFS mounts on management server during server starup by correcting how mount points are listed for a management server. (cherry picked from commit 647532376fc090f75e6fa0ad66ee36241084dec9) Signed-off-by: Rohit Yadav --- core/src/com/cloud/storage/JavaStorageLayer.java | 14 ++++++++++++++ core/src/com/cloud/storage/StorageLayer.java | 3 +++ .../vmware/manager/VmwareManagerImpl.java | 4 ++-- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/core/src/com/cloud/storage/JavaStorageLayer.java b/core/src/com/cloud/storage/JavaStorageLayer.java index 6b61ad058e4..2e5544c8c19 100644 --- a/core/src/com/cloud/storage/JavaStorageLayer.java +++ b/core/src/com/cloud/storage/JavaStorageLayer.java @@ -142,6 +142,20 @@ public class JavaStorageLayer implements StorageLayer { return paths; } + @Override + public List listMountPointsByMsHost(String path, long msHostId) { + List mountPaths = new ArrayList(); + File[] files = new File(path).listFiles(); + if (files == null) { + return mountPaths; + } + for (File file : files) { + if (file.getName().startsWith(String.valueOf(msHostId) + ".")) + mountPaths.add(file.getAbsolutePath()); + } + return mountPaths; + } + @Override public boolean mkdir(String path) { synchronized (path.intern()) { diff --git a/core/src/com/cloud/storage/StorageLayer.java b/core/src/com/cloud/storage/StorageLayer.java index 8daa392ef9e..8421aeb1285 100644 --- a/core/src/com/cloud/storage/StorageLayer.java +++ b/core/src/com/cloud/storage/StorageLayer.java @@ -21,6 +21,7 @@ package com.cloud.storage; import java.io.File; import java.io.IOException; +import java.util.List; import com.cloud.utils.component.Manager; @@ -149,4 +150,6 @@ public interface StorageLayer extends Manager { boolean setWorldReadableAndWriteable(File file); boolean deleteDir(String dir); + + List listMountPointsByMsHost(String path, long msHostId); } diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java index 0e67a85cd30..f6205d5e8b2 100755 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java @@ -696,8 +696,8 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw long mshostId = ManagementServerNode.getManagementServerId(); // cleanup left-over NFS mounts from previous session - String[] mounts = _storage.listFiles(parent + File.separator + String.valueOf(mshostId) + ".*"); - if (mounts != null && mounts.length > 0) { + List mounts = _storage.listMountPointsByMsHost(parent, mshostId); + if (mounts != null && !mounts.isEmpty()) { for (String mountPoint : mounts) { s_logger.info("umount NFS mount from previous session: " + mountPoint); From 86fe713ab9ca40f0be1b0d6a8142352a54dc6079 Mon Sep 17 00:00:00 2001 From: Anthony Xu Date: Tue, 21 Oct 2014 11:14:15 -0700 Subject: [PATCH 140/391] don't need to check if pv driver is updated, old pv driver should work (cherry picked from commit 45611a0f83024d055adcce0d05b5fcfb567e329b) Signed-off-by: Rohit Yadav --- .../xenserver/resource/CitrixResourceBase.java | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java b/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java index ee4e3d534de..ef1a9506613 100644 --- a/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java +++ b/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java @@ -80,7 +80,6 @@ import com.xensource.xenapi.VGPU; import com.xensource.xenapi.VIF; import com.xensource.xenapi.VLAN; import com.xensource.xenapi.VM; -import com.xensource.xenapi.VMGuestMetrics; import com.xensource.xenapi.XenAPIObject; import org.apache.cloudstack.storage.command.StorageSubSystemCommand; @@ -6148,23 +6147,6 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe // Look up the VM VM vm = getVM(conn, vmName); - /* For HVM guest, if no pv driver installed, no attach/detach */ - boolean isHVM; - if (vm.getPVBootloader(conn).equalsIgnoreCase("")) { - isHVM = true; - } else { - isHVM = false; - } - VMGuestMetrics vgm = vm.getGuestMetrics(conn); - boolean pvDrvInstalled = false; - if (!isRefNull(vgm) && vgm.getPVDriversUpToDate(conn)) { - pvDrvInstalled = true; - } - if (isHVM && !pvDrvInstalled) { - s_logger.warn(errorMsg + ": You attempted an operation on a VM which requires PV drivers to be installed but the drivers were not detected"); - return new AttachVolumeAnswer(cmd, - "You attempted an operation that requires PV drivers to be installed on the VM. Please install them by inserting xen-pv-drv.iso."); - } if (attach) { // Figure out the disk number to attach the VM to String diskNumber = null; From 64e938b7014da9994185dfe80120254cd96d4f7a Mon Sep 17 00:00:00 2001 From: Anthony Xu Date: Mon, 13 Oct 2014 10:29:58 -0700 Subject: [PATCH 141/391] fixed NPE on calculating vm snasphot volume size (cherry picked from commit 95c7ffbd46f9f674b8813a7b992c17019eb00137) Signed-off-by: Rohit Yadav --- .../xenserver/resource/CitrixResourceBase.java | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java b/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java index ef1a9506613..a90f99d0fc2 100644 --- a/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java +++ b/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java @@ -6232,11 +6232,11 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe // add size of snapshot vdi node, usually this only contains meta data size = size + vdi.getPhysicalUtilisation(conn); // add size of snapshot vdi parent, this contains data - if (parentVDI != null) + if (!isRefNull(parentVDI)) size = size + parentVDI.getPhysicalUtilisation(conn).longValue(); } } catch (Exception e) { - s_logger.debug("Exception occurs when calculate " + "snapshot capacity for volumes: " + e.getMessage()); + s_logger.debug("Exception occurs when calculate snapshot capacity for volumes: due to " + e.toString()); continue; } } @@ -6248,13 +6248,17 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe try { String vName = vmr.getNameLabel(conn); if (vName != null && vName.contains(vmName) && vmr.getIsASnapshot(conn)) { - VDI memoryVDI = vmr.getSuspendVDI(conn); - size = size + memoryVDI.getParent(conn).getPhysicalUtilisation(conn); - size = size + memoryVDI.getPhysicalUtilisation(conn); + if (!isRefNull(memoryVDI)) { + size = size + memoryVDI.getPhysicalUtilisation(conn); + VDI pMemoryVDI = memoryVDI.getParent(conn); + if (!isRefNull(pMemoryVDI)) { + size = size + pMemoryVDI.getPhysicalUtilisation(conn); + } + } } } catch (Exception e) { - s_logger.debug("Exception occurs when calculate " + "snapshot capacity for memory: " + e.getMessage()); + s_logger.debug("Exception occurs when calculate snapshot capacity for memory: due to " + e.toString()); continue; } } From 48ec9ac11b0772ffb9b7be4947f26aba0ad9b89d Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Mon, 2 Feb 2015 16:09:56 +0530 Subject: [PATCH 142/391] CLOUDSTACK-8190: Fix xenserver traffic label for backward compatiblity Signed-off-by: Rohit Yadav --- .../apache/cloudstack/api/ApiConstants.java | 2 +- ui/scripts/system.js | 24 +++++++++---------- ui/scripts/zoneWizard.js | 2 +- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/api/src/org/apache/cloudstack/api/ApiConstants.java b/api/src/org/apache/cloudstack/api/ApiConstants.java index 6bd5ddede3e..b6aed6f80fc 100755 --- a/api/src/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/org/apache/cloudstack/api/ApiConstants.java @@ -380,7 +380,7 @@ public class ApiConstants { public static final String SECURITY_GROUP_EANBLED = "securitygroupenabled"; public static final String LOCAL_STORAGE_ENABLED = "localstorageenabled"; public static final String GUEST_IP_TYPE = "guestiptype"; - public static final String XENSERVER_NETWORK_LABEL = "xenservernetworklabel"; + public static final String XENSERVER_NETWORK_LABEL = "xennetworklabel"; public static final String KVM_NETWORK_LABEL = "kvmnetworklabel"; public static final String VMWARE_NETWORK_LABEL = "vmwarenetworklabel"; public static final String HYPERV_NETWORK_LABEL = "hypervnetworklabel"; diff --git a/ui/scripts/system.js b/ui/scripts/system.js index f55807afe32..3893dc75e62 100644 --- a/ui/scripts/system.js +++ b/ui/scripts/system.js @@ -127,8 +127,8 @@ } }); - if (trafficType.xenservernetworklabel == null || trafficType.xenservernetworklabel == 0) - trafficType.xenservernetworklabel = dictionary[ 'label.network.label.display.for.blank.value']; + if (trafficType.xennetworklabel == null || trafficType.xennetworklabel == 0) + trafficType.xennetworklabel = dictionary[ 'label.network.label.display.for.blank.value']; if (trafficType.kvmnetworklabel == null || trafficType.kvmnetworklabel == 0) trafficType.kvmnetworklabel = dictionary[ 'label.network.label.display.for.blank.value']; if (trafficType.vmwarenetworklabel == null || trafficType.vmwarenetworklabel == 0) @@ -145,8 +145,8 @@ var updateTrafficLabels = function (trafficType, labels, complete) { var array1 =[]; - if (labels.xenservernetworklabel != dictionary[ 'label.network.label.display.for.blank.value']) - array1.push("&xenservernetworklabel=" + labels.xenservernetworklabel); + if (labels.xennetworklabel != dictionary[ 'label.network.label.display.for.blank.value']) + array1.push("&xennetworklabel=" + labels.xennetworklabel); if (labels.kvmnetworklabel != dictionary[ 'label.network.label.display.for.blank.value']) array1.push("&kvmnetworklabel=" + labels.kvmnetworklabel); if (labels.vmwarenetworklabel != dictionary[ 'label.network.label.display.for.blank.value']) @@ -573,7 +573,7 @@ } }, { - xenservernetworklabel: { + xennetworklabel: { label: 'label.xenserver.traffic.label', isEditable: true }, @@ -613,7 +613,7 @@ selectedPublicNetworkObj = items[0]; // Include traffic labels - selectedPublicNetworkObj.xenservernetworklabel = trafficType.xenservernetworklabel; + selectedPublicNetworkObj.xennetworklabel = trafficType.xennetworklabel; selectedPublicNetworkObj.kvmnetworklabel = trafficType.kvmnetworklabel; selectedPublicNetworkObj.vmwarenetworklabel = trafficType.vmwarenetworklabel; selectedPublicNetworkObj.ovmnetworklabel = trafficType.ovmnetworklabel; @@ -883,7 +883,7 @@ } }, { - xenservernetworklabel: { + xennetworklabel: { label: 'label.xenserver.traffic.label', isEditable: true }, @@ -919,7 +919,7 @@ var trafficType = getTrafficType(selectedPhysicalNetworkObj, 'Storage'); selectedPublicNetworkObj = items[0]; - selectedPublicNetworkObj.xenservernetworklabel = trafficType.xenservernetworklabel; + selectedPublicNetworkObj.xennetworklabel = trafficType.xennetworklabel; selectedPublicNetworkObj.kvmnetworklabel = trafficType.kvmnetworklabel; selectedPublicNetworkObj.vmwarenetworklabel = trafficType.vmwarenetworklabel; selectedPublicNetworkObj.ovmnetworklabel = trafficType.ovmnetworklabel; @@ -1095,7 +1095,7 @@ } }, { - xenservernetworklabel: { + xennetworklabel: { label: 'label.xenserver.traffic.label', isEditable: true }, @@ -1129,7 +1129,7 @@ var trafficType = getTrafficType(selectedPhysicalNetworkObj, 'Management'); - selectedManagementNetworkObj.xenservernetworklabel = trafficType.xenservernetworklabel; + selectedManagementNetworkObj.xennetworklabel = trafficType.xennetworklabel; selectedManagementNetworkObj.kvmnetworklabel = trafficType.kvmnetworklabel; selectedManagementNetworkObj.vmwarenetworklabel = trafficType.vmwarenetworklabel; selectedManagementNetworkObj.ovmnetworklabel = trafficType.ovmnetworklabel; @@ -1284,7 +1284,7 @@ }, { //updateTrafficType API - xenservernetworklabel: { + xennetworklabel: { label: 'label.xenserver.traffic.label', isEditable: true }, @@ -1340,7 +1340,7 @@ var xenservertrafficlabel, kvmtrafficlabel, vmwaretrafficlabel; var trafficType = getTrafficType(selectedPhysicalNetworkObj, 'Guest'); //refresh Guest traffic type - selectedPhysicalNetworkObj[ "xenservernetworklabel"] = trafficType.xenservernetworklabel; + selectedPhysicalNetworkObj[ "xennetworklabel"] = trafficType.xennetworklabel; selectedPhysicalNetworkObj[ "kvmnetworklabel"] = trafficType.kvmnetworklabel; selectedPhysicalNetworkObj[ "vmwarenetworklabel"] = trafficType.vmwarenetworklabel; selectedPhysicalNetworkObj[ "ovmnetworklabel"] = trafficType.ovmnetworklabel; diff --git a/ui/scripts/zoneWizard.js b/ui/scripts/zoneWizard.js index 299eff25a30..66b34699eb0 100755 --- a/ui/scripts/zoneWizard.js +++ b/ui/scripts/zoneWizard.js @@ -71,7 +71,7 @@ switch (hypervisor) { case 'XenServer': - hypervisorAttr = 'xenservernetworklabel'; + hypervisorAttr = 'xennetworklabel'; break; case 'KVM': hypervisorAttr = 'kvmnetworklabel'; From 876c78fe1ba6abe132131b3449b21fd09f2c14e1 Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Mon, 2 Feb 2015 19:56:25 +0530 Subject: [PATCH 143/391] CLOUDSTACK-8191: SAML users should have their own accounts Signed-off-by: Rohit Yadav --- .../classes/resources/messages.properties | 1 + .../resources/messages_de_DE.properties | 1 + .../classes/resources/messages_es.properties | 1 + .../resources/messages_fr_FR.properties | 1 + .../resources/messages_ja_JP.properties | 1 + .../resources/messages_ko_KR.properties | 1 + .../resources/messages_nl_NL.properties | 1 + .../classes/resources/messages_pl.properties | 1 + .../resources/messages_pt_BR.properties | 1 + .../resources/messages_ru_RU.properties | 1 + .../resources/messages_zh_CN.properties | 1 + .../SAML2LoginAPIAuthenticatorCmd.java | 42 +++++++++++-------- .../SAML2LoginAPIAuthenticatorCmdTest.java | 28 ++++++------- .../src/com/cloud/configuration/Config.java | 8 ---- .../com/cloud/user/AccountManagerImpl.java | 2 +- ui/css/cloudstack3.css | 24 ++++++++++- ui/dictionary.jsp | 1 + ui/index.jsp | 1 + ui/scripts/cloudStack.js | 10 +++++ ui/scripts/ui-custom/login.js | 6 +++ 20 files changed, 90 insertions(+), 43 deletions(-) diff --git a/client/WEB-INF/classes/resources/messages.properties b/client/WEB-INF/classes/resources/messages.properties index 51faf82a73f..234e4af9abd 100644 --- a/client/WEB-INF/classes/resources/messages.properties +++ b/client/WEB-INF/classes/resources/messages.properties @@ -755,6 +755,7 @@ label.local.storage=Local Storage label.local=Local label.login=Login label.logout=Logout +label.saml.login=SAML Login label.LUN.number=LUN \# label.lun=LUN label.make.project.owner=Make account project owner diff --git a/client/WEB-INF/classes/resources/messages_de_DE.properties b/client/WEB-INF/classes/resources/messages_de_DE.properties index 616cdf9048f..bd360882ef7 100644 --- a/client/WEB-INF/classes/resources/messages_de_DE.properties +++ b/client/WEB-INF/classes/resources/messages_de_DE.properties @@ -572,6 +572,7 @@ label.local=Lokal label.local.storage=Lokaler Speicher label.login=Login label.logout=Abmelden +label.saml.login=SAML Login label.lun=LUN label.LUN.number=LUN \# label.management=Verwaltung diff --git a/client/WEB-INF/classes/resources/messages_es.properties b/client/WEB-INF/classes/resources/messages_es.properties index 6560a2a5513..6b51c38bf38 100644 --- a/client/WEB-INF/classes/resources/messages_es.properties +++ b/client/WEB-INF/classes/resources/messages_es.properties @@ -555,6 +555,7 @@ label.local=local label.local.storage=Almacenamiento Local label.login=Login label.logout=Cerrar sesi\u00c3\u00b3n +label.saml.login=SAML Login label.lun=LUN label.LUN.number=LUN \# label.manage=Administrar diff --git a/client/WEB-INF/classes/resources/messages_fr_FR.properties b/client/WEB-INF/classes/resources/messages_fr_FR.properties index 54dc6215a83..1272442acdb 100644 --- a/client/WEB-INF/classes/resources/messages_fr_FR.properties +++ b/client/WEB-INF/classes/resources/messages_fr_FR.properties @@ -878,6 +878,7 @@ label.local.storage.enabled=Stockage local activ\u00e9 label.local.storage=Stockage local label.login=Connexion label.logout=D\u00e9connexion +label.saml.login=SAML Connexion label.lun=LUN label.LUN.number=N\u00b0 LUN label.lxc.traffic.label=Libell\u00e9 trafic LXC diff --git a/client/WEB-INF/classes/resources/messages_ja_JP.properties b/client/WEB-INF/classes/resources/messages_ja_JP.properties index 85d211fdf78..7a00ab70dab 100644 --- a/client/WEB-INF/classes/resources/messages_ja_JP.properties +++ b/client/WEB-INF/classes/resources/messages_ja_JP.properties @@ -878,6 +878,7 @@ label.local.storage=\u30ed\u30fc\u30ab\u30eb \u30b9\u30c8\u30ec\u30fc\u30b8 label.local=\u30ed\u30fc\u30ab\u30eb label.login=\u30ed\u30b0\u30aa\u30f3 label.logout=\u30ed\u30b0\u30aa\u30d5 +label.saml.login=SAML \u30ed\u30b0\u30aa\u30f3 label.lun=LUN label.LUN.number=LUN \u756a\u53f7 label.lxc.traffic.label=LXC \u306e\u30c8\u30e9\u30d5\u30a3\u30c3\u30af \u30e9\u30d9\u30eb diff --git a/client/WEB-INF/classes/resources/messages_ko_KR.properties b/client/WEB-INF/classes/resources/messages_ko_KR.properties index 42ffe6c9760..badb7f3b737 100644 --- a/client/WEB-INF/classes/resources/messages_ko_KR.properties +++ b/client/WEB-INF/classes/resources/messages_ko_KR.properties @@ -651,6 +651,7 @@ label.local.storage=\ub85c\uceec \uc2a4\ud1a0\ub9ac\uc9c0 label.local=\ub85c\uceec label.login=\ub85c\uadf8\uc778 label.logout=\ub85c\uadf8\uc544\uc6c3 +label.saml.login=SAML \ub85c\uadf8\uc778 label.lun=LUN label.LUN.number=LUN \ubc88\ud638 label.make.project.owner=\uacc4\uc815 \uc815\ubcf4 \ud504\ub85c\uc81d\ud2b8 \uc18c\uc720\uc790 diff --git a/client/WEB-INF/classes/resources/messages_nl_NL.properties b/client/WEB-INF/classes/resources/messages_nl_NL.properties index 53c1dc4a1fc..35cd86c92b9 100644 --- a/client/WEB-INF/classes/resources/messages_nl_NL.properties +++ b/client/WEB-INF/classes/resources/messages_nl_NL.properties @@ -826,6 +826,7 @@ label.local.storage.enabled=Lokale opslag ingeschakeld label.local.storage=Lokale Opslag label.login=Login label.logout=Log uit +label.saml.login=SAML Login label.lun=LUN label.LUN.number=LUN \# label.lxc.traffic.label=LXC verkeerslabel diff --git a/client/WEB-INF/classes/resources/messages_pl.properties b/client/WEB-INF/classes/resources/messages_pl.properties index a0d1b354b4d..0388b3ff7e3 100644 --- a/client/WEB-INF/classes/resources/messages_pl.properties +++ b/client/WEB-INF/classes/resources/messages_pl.properties @@ -292,6 +292,7 @@ label.local.storage.enabled=Pami\u0119\u0107 lokalna w\u0142\u0105czona label.local.storage=Pami\u0119\u0107 lokalna label.login=Zaloguj label.logout=Wyloguj +label.saml.login=SAML Zaloguj label.lun=LUN label.LUN.number=LUN \# label.max.guest.limit=Maksymalna liczba go\u015bci diff --git a/client/WEB-INF/classes/resources/messages_pt_BR.properties b/client/WEB-INF/classes/resources/messages_pt_BR.properties index d434fef46ad..c925e6d090b 100644 --- a/client/WEB-INF/classes/resources/messages_pt_BR.properties +++ b/client/WEB-INF/classes/resources/messages_pt_BR.properties @@ -728,6 +728,7 @@ label.local.storage.enabled=Storage local habilitada label.local.storage=Storage Local label.login=Entrar label.logout=Sair +label.saml.login=SAML Entrar label.lun=LUN label.LUN.number=LUN \# label.make.project.owner=Criar propriet\u00e1rio de conta de projeto diff --git a/client/WEB-INF/classes/resources/messages_ru_RU.properties b/client/WEB-INF/classes/resources/messages_ru_RU.properties index 22b6c075adb..16427a1e9b0 100644 --- a/client/WEB-INF/classes/resources/messages_ru_RU.properties +++ b/client/WEB-INF/classes/resources/messages_ru_RU.properties @@ -689,6 +689,7 @@ label.local.storage=\u041b\u043e\u043a\u0430\u043b\u044c\u043d\u043e\u0435 \u044 label.local=\u041b\u043e\u043a\u0430\u043b\u044c\u043d\u044b\u0439 label.login=\u0412\u0445\u043e\u0434 label.logout=\u0412\u044b\u0445\u043e\u0434 +label.saml.login=SAML \u0412\u0445\u043e\u0434 label.lun=LUN label.LUN.number=LUN \# label.make.project.owner=\u0421\u0434\u0435\u043b\u0430\u0442\u044c \u0430\u043a\u043a\u0430\u0443\u043d\u0442 \u0432\u043b\u0430\u0434\u0435\u043b\u044c\u0446\u0435\u043c \u043f\u0440\u043e\u0435\u043a\u0442\u0430 diff --git a/client/WEB-INF/classes/resources/messages_zh_CN.properties b/client/WEB-INF/classes/resources/messages_zh_CN.properties index cedd94f51a2..289c3a8c9ed 100644 --- a/client/WEB-INF/classes/resources/messages_zh_CN.properties +++ b/client/WEB-INF/classes/resources/messages_zh_CN.properties @@ -878,6 +878,7 @@ label.local.storage=\u672c\u5730\u5b58\u50a8 label.local=\u672c\u5730 label.login=\u767b\u5f55 label.logout=\u6ce8\u9500 +label.saml.login=SAML \u767b\u5f55 label.lun=LUN label.LUN.number=LUN \u53f7 label.lxc.traffic.label=LXC \u6d41\u91cf\u6807\u7b7e diff --git a/plugins/user-authenticators/saml2/src/org/apache/cloudstack/api/command/SAML2LoginAPIAuthenticatorCmd.java b/plugins/user-authenticators/saml2/src/org/apache/cloudstack/api/command/SAML2LoginAPIAuthenticatorCmd.java index a0dcad5c9cb..27354a40649 100644 --- a/plugins/user-authenticators/saml2/src/org/apache/cloudstack/api/command/SAML2LoginAPIAuthenticatorCmd.java +++ b/plugins/user-authenticators/saml2/src/org/apache/cloudstack/api/command/SAML2LoginAPIAuthenticatorCmd.java @@ -23,7 +23,8 @@ import com.cloud.domain.Domain; import com.cloud.exception.CloudAuthenticationException; import com.cloud.user.Account; import com.cloud.user.DomainManager; -import com.cloud.user.User; +import com.cloud.user.UserAccount; +import com.cloud.user.dao.UserAccountDao; import com.cloud.utils.HttpUtils; import com.cloud.utils.db.EntityManager; import org.apache.cloudstack.api.APICommand; @@ -72,6 +73,7 @@ import java.security.NoSuchAlgorithmException; import java.security.PrivateKey; import java.util.List; import java.util.Map; +import java.util.UUID; @APICommand(name = "samlSso", description = "SP initiated SAML Single Sign On", requestHasSensitiveInfo = true, responseObject = LoginCmdResponse.class, entityType = {}) public class SAML2LoginAPIAuthenticatorCmd extends BaseCmd implements APIAuthenticator { @@ -92,6 +94,8 @@ public class SAML2LoginAPIAuthenticatorCmd extends BaseCmd implements APIAuthent ConfigurationDao _configDao; @Inject DomainManager _domainMgr; + @Inject + private UserAccountDao _userAccountDao; SAML2AuthManager _samlAuthManager; @@ -200,11 +204,9 @@ public class SAML2LoginAPIAuthenticatorCmd extends BaseCmd implements APIAuthent } } - String uniqueUserId = null; - String accountName = _configDao.getValue(Config.SAMLUserAccountName.key()); String domainString = _configDao.getValue(Config.SAMLUserDomain.key()); - Long domainId = -1L; + Long domainId = null; Domain domain = _domainMgr.getDomain(domainString); if (domain != null) { domainId = domain.getId(); @@ -214,16 +216,17 @@ public class SAML2LoginAPIAuthenticatorCmd extends BaseCmd implements APIAuthent } catch (NumberFormatException ignore) { } } - if (domainId == -1L) { - s_logger.error("The default domain ID for SAML users is not set correct, it should be a UUID"); + if (domainId == null) { + s_logger.error("The default domain ID for SAML users is not set correct, it should be a UUID. ROOT domain will be used."); } String username = null; String password = SAMLUtils.generateSecureRandomId(); // Random password String firstName = ""; String lastName = ""; - String timeZone = ""; + String timeZone = "GMT"; String email = ""; + short accountType = 0; // User account Assertion assertion = processedSAMLResponse.getAssertions().get(0); NameID nameId = assertion.getSubject().getNameID(); @@ -233,7 +236,6 @@ public class SAML2LoginAPIAuthenticatorCmd extends BaseCmd implements APIAuthent if (nameId.getFormat().equals(NameIDType.PERSISTENT) || nameId.getFormat().equals(NameIDType.EMAIL)) { username = nameId.getValue(); - uniqueUserId = SAMLUtils.createSAMLId(username); if (nameId.getFormat().equals(NameIDType.EMAIL)) { email = username; } @@ -249,9 +251,8 @@ public class SAML2LoginAPIAuthenticatorCmd extends BaseCmd implements APIAuthent for (Attribute attribute: attributeStatement.getAttributes()) { String attributeName = attribute.getName(); String attributeValue = attribute.getAttributeValues().get(0).getDOM().getTextContent(); - if (attributeName.equalsIgnoreCase("uid") && uniqueUserId == null) { + if (attributeName.equalsIgnoreCase("uid") && username == null) { username = attributeValue; - uniqueUserId = SAMLUtils.createSAMLId(username); } else if (attributeName.equalsIgnoreCase("givenName")) { firstName = attributeValue; } else if (attributeName.equalsIgnoreCase(("sn"))) { @@ -263,17 +264,22 @@ public class SAML2LoginAPIAuthenticatorCmd extends BaseCmd implements APIAuthent } } - User user = _entityMgr.findByUuid(User.class, uniqueUserId); - if (user == null && uniqueUserId != null && username != null - && accountName != null && domainId != null) { - CallContext.current().setEventDetails("UserName: " + username + ", FirstName :" + password + ", LastName: " + lastName); - user = _accountService.createUser(username, password, firstName, lastName, email, timeZone, accountName, domainId, uniqueUserId); + if (username == null && email != null) { + username = email; + } + final String uniqueUserId = SAMLUtils.createSAMLId(username); + + UserAccount userAccount = _userAccountDao.getUserAccount(username, domainId); + if (userAccount == null && uniqueUserId != null && username != null) { + CallContext.current().setEventDetails("SAML Account/User with UserName: " + username + ", FirstName :" + password + ", LastName: " + lastName); + _accountService.createUserAccount(username, password, firstName, lastName, email, timeZone, + username, (short) accountType, domainId, null, null, UUID.randomUUID().toString(), uniqueUserId); } - if (user != null) { + if (userAccount != null) { try { - if (_apiServer.verifyUser(user.getId())) { - LoginCmdResponse loginResponse = (LoginCmdResponse) _apiServer.loginUser(session, username, user.getPassword(), domainId, null, remoteAddress, params); + if (_apiServer.verifyUser(userAccount.getId())) { + LoginCmdResponse loginResponse = (LoginCmdResponse) _apiServer.loginUser(session, username, userAccount.getPassword(), domainId, null, remoteAddress, params); resp.addCookie(new Cookie("userid", URLEncoder.encode(loginResponse.getUserId(), HttpUtils.UTF_8))); resp.addCookie(new Cookie("domainid", URLEncoder.encode(loginResponse.getDomainId(), HttpUtils.UTF_8))); resp.addCookie(new Cookie("role", URLEncoder.encode(loginResponse.getType(), HttpUtils.UTF_8))); diff --git a/plugins/user-authenticators/saml2/test/org/apache/cloudstack/api/command/SAML2LoginAPIAuthenticatorCmdTest.java b/plugins/user-authenticators/saml2/test/org/apache/cloudstack/api/command/SAML2LoginAPIAuthenticatorCmdTest.java index 514edb507d9..b12d43202ff 100644 --- a/plugins/user-authenticators/saml2/test/org/apache/cloudstack/api/command/SAML2LoginAPIAuthenticatorCmdTest.java +++ b/plugins/user-authenticators/saml2/test/org/apache/cloudstack/api/command/SAML2LoginAPIAuthenticatorCmdTest.java @@ -22,10 +22,9 @@ package org.apache.cloudstack.api.command; import com.cloud.domain.Domain; import com.cloud.user.AccountService; import com.cloud.user.DomainManager; -import com.cloud.user.User; -import com.cloud.user.UserVO; +import com.cloud.user.UserAccountVO; +import com.cloud.user.dao.UserAccountDao; import com.cloud.utils.HttpUtils; -import com.cloud.utils.db.EntityManager; import org.apache.cloudstack.api.ApiServerService; import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.ServerApiException; @@ -78,15 +77,15 @@ public class SAML2LoginAPIAuthenticatorCmdTest { @Mock ConfigurationDao configDao; - @Mock - EntityManager entityMgr; - @Mock DomainManager domainMgr; @Mock AccountService accountService; + @Mock + UserAccountDao userAccountDao; + @Mock Domain domain; @@ -138,10 +137,6 @@ public class SAML2LoginAPIAuthenticatorCmdTest { accountServiceField.setAccessible(true); accountServiceField.set(cmd, accountService); - Field entityMgrField = SAML2LoginAPIAuthenticatorCmd.class.getDeclaredField("_entityMgr"); - entityMgrField.setAccessible(true); - entityMgrField.set(cmd, entityMgr); - Field domainMgrField = SAML2LoginAPIAuthenticatorCmd.class.getDeclaredField("_domainMgr"); domainMgrField.setAccessible(true); domainMgrField.set(cmd, domainMgr); @@ -150,6 +145,10 @@ public class SAML2LoginAPIAuthenticatorCmdTest { configDaoField.setAccessible(true); configDaoField.set(cmd, configDao); + Field userAccountDaoField = SAML2LoginAPIAuthenticatorCmd.class.getDeclaredField("_userAccountDao"); + userAccountDaoField.setAccessible(true); + userAccountDaoField.set(cmd, userAccountDao); + String spId = "someSPID"; String url = "someUrl"; X509Certificate cert = SAMLUtils.generateRandomX509Certificate(SAMLUtils.generateRandomKeyPair()); @@ -163,9 +162,10 @@ public class SAML2LoginAPIAuthenticatorCmdTest { Mockito.when(domain.getId()).thenReturn(1L); Mockito.when(domainMgr.getDomain(Mockito.anyString())).thenReturn(domain); - UserVO user = new UserVO(); - user.setUuid(SAMLUtils.createSAMLId("someUID")); - Mockito.when(entityMgr.findByUuid(Mockito.eq(User.class), Mockito.anyString())).thenReturn((User) user); + UserAccountVO user = new UserAccountVO(); + user.setUsername(SAMLUtils.createSAMLId("someUID")); + user.setId(1000L); + Mockito.when(userAccountDao.getUserAccount(Mockito.anyString(), Mockito.anyLong())).thenReturn(user); Mockito.when(apiServer.verifyUser(Mockito.anyLong())).thenReturn(false); Map params = new HashMap(); @@ -183,7 +183,7 @@ public class SAML2LoginAPIAuthenticatorCmdTest { } Mockito.verify(configDao, Mockito.atLeastOnce()).getValue(Mockito.anyString()); Mockito.verify(domainMgr, Mockito.times(1)).getDomain(Mockito.anyString()); - Mockito.verify(entityMgr, Mockito.times(1)).findByUuid(Mockito.eq(User.class), Mockito.anyString()); + Mockito.verify(userAccountDao, Mockito.times(1)).getUserAccount(Mockito.anyString(), Mockito.anyLong()); Mockito.verify(apiServer, Mockito.times(1)).verifyUser(Mockito.anyLong()); } diff --git a/server/src/com/cloud/configuration/Config.java b/server/src/com/cloud/configuration/Config.java index dfb5f2aa8fc..93ee1ca8e50 100755 --- a/server/src/com/cloud/configuration/Config.java +++ b/server/src/com/cloud/configuration/Config.java @@ -1378,14 +1378,6 @@ public enum Config { "false", "Set it to true to enable SAML SSO plugin", null), - SAMLUserAccountName( - "Advanced", - ManagementServer.class, - String.class, - "saml2.default.accountname", - "admin", - "The name of the default account to use when creating users from SAML SSO", - null), SAMLUserDomain( "Advanced", ManagementServer.class, diff --git a/server/src/com/cloud/user/AccountManagerImpl.java b/server/src/com/cloud/user/AccountManagerImpl.java index 6e9e0d72928..1fc288fcd5c 100755 --- a/server/src/com/cloud/user/AccountManagerImpl.java +++ b/server/src/com/cloud/user/AccountManagerImpl.java @@ -1014,7 +1014,7 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M throw new InvalidParameterValueException("The user " + userName + " already exists in domain " + domainId); } - if (networkDomain != null) { + if (networkDomain != null && networkDomain.length() > 0) { if (!NetUtils.verifyDomainName(networkDomain)) { throw new InvalidParameterValueException( "Invalid network domain. Total length shouldn't exceed 190 chars. Each domain label must be between 1 and 63 characters long, can contain ASCII letters 'a' through 'z', the digits '0' through '9', " diff --git a/ui/css/cloudstack3.css b/ui/css/cloudstack3.css index b3f9b069f80..70a8abfd638 100644 --- a/ui/css/cloudstack3.css +++ b/ui/css/cloudstack3.css @@ -440,7 +440,7 @@ body.login { background: transparent url(../images/sprites.png) -563px -747px; cursor: pointer; border: none; - margin: 7px 238px 0 -1px; + margin: 7px 120px 0 -1px; text-align: center; width: 69px; height: 25px; @@ -456,6 +456,27 @@ body.login { text-shadow: 0px 1px 2px #000000; } +.login .fields input[type=samlsubmit] { + background: transparent url(../images/sprites.png) -563px -747px; + cursor: pointer; + border: none; + margin: 7px 120px 0 -1px; + text-align: center; + width: 60px; + height: 15px; + display: block; + color: #FFFFFF; + font-weight: bold; + float: left; + text-indent: -1px; + /*+text-shadow:0px 1px 2px #000000;*/ + -moz-text-shadow: 0px 1px 2px #000000; + -webkit-text-shadow: 0px 1px 2px #000000; + -o-text-shadow: 0px 1px 2px #000000; + text-shadow: 0px 1px 2px #000000; + font-size: 10px; +} + .login .fields input[type=submit]:hover { background-position: -563px -772px; } @@ -13017,4 +13038,3 @@ div.gpugroups div.list-view { background: transparent url("../images/icons.png") no-repeat -626px -209px; padding: 0 0 3px 18px; } - diff --git a/ui/dictionary.jsp b/ui/dictionary.jsp index 272cc260a08..c7599ed1c40 100644 --- a/ui/dictionary.jsp +++ b/ui/dictionary.jsp @@ -760,6 +760,7 @@ dictionary = { 'label.local.storage': '', 'label.login': '', 'label.logout': '', +'label.saml.login': '', 'label.lun': '', 'label.LUN.number': '', 'label.make.project.owner': '', diff --git a/ui/index.jsp b/ui/index.jsp index fc2a742e2af..a02f944bea7 100644 --- a/ui/index.jsp +++ b/ui/index.jsp @@ -67,6 +67,7 @@ " /> + " />
" /> - " /> +
"/>